2024-08-31T02:10:34.453 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-08-31T02:10:34.477 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-31T02:10:34.594 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7882562' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: 34b9d27bc925f169ace27b9f1f204536aa49fc0b ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 34b9d27bc925f169ace27b9f1f204536aa49fc0b workunit: branch: squid sha1: 244a32d3583bb0aa259286ff61157888a54a3ca5 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 6007 sha1: 34b9d27bc925f169ace27b9f1f204536aa49fc0b sleep_before_teardown: 0 subset: 49/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 244a32d3583bb0aa259286ff61157888a54a3ca5 targets: smithi070.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKCONr/3gTV3cdORYrr5VYidYAdvCOHBoKnpVkekwRNFb0JQlRuwwYmF/aHJt6n1wSlHkvpUNN89LgJTIhaaZcw= smithi102.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFYYmdiKiEnUt6QsSXqLW997tDcGyJ8Q1ec2uuQNrKPNCvoluEhAkYa+9oJ20uODo5aH4bQvGxVqQ29JKPSusoo= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-08-30_21:08:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-31T02:10:34.595 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa; will attempt to use it 2024-08-31T02:10:34.595 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa/tasks 2024-08-31T02:10:34.596 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-31T02:10:34.597 INFO:teuthology.task.internal:Checking packages... 2024-08-31T02:10:34.618 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '34b9d27bc925f169ace27b9f1f204536aa49fc0b' 2024-08-31T02:10:34.618 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-31T02:10:34.618 INFO:teuthology.packaging:ref: None 2024-08-31T02:10:34.618 INFO:teuthology.packaging:tag: None 2024-08-31T02:10:34.618 INFO:teuthology.packaging:branch: squid 2024-08-31T02:10:34.618 INFO:teuthology.packaging:sha1: 34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:10:34.619 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-08-31T02:10:34.852 INFO:teuthology.task.internal:Found packages for ceph version 19.1.1-242.g244a32d3 2024-08-31T02:10:34.854 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-31T02:10:34.859 INFO:teuthology.task.internal:no buildpackages task found 2024-08-31T02:10:34.860 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-31T02:10:34.868 INFO:teuthology.task.internal:Saving configuration 2024-08-31T02:10:34.890 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-31T02:10:34.896 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-31T02:10:34.923 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-31 02:05:43.529975', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKCONr/3gTV3cdORYrr5VYidYAdvCOHBoKnpVkekwRNFb0JQlRuwwYmF/aHJt6n1wSlHkvpUNN89LgJTIhaaZcw='} 2024-08-31T02:10:34.945 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi102.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-31 02:05:43.528564', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFYYmdiKiEnUt6QsSXqLW997tDcGyJ8Q1ec2uuQNrKPNCvoluEhAkYa+9oJ20uODo5aH4bQvGxVqQ29JKPSusoo='} 2024-08-31T02:10:34.945 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-31T02:10:34.951 INFO:teuthology.task.internal:roles: ubuntu@smithi070.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-31T02:10:34.951 INFO:teuthology.task.internal:roles: ubuntu@smithi102.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-31T02:10:34.951 INFO:teuthology.run_tasks:Running task console_log... 2024-08-31T02:10:35.016 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4c04d37ac0>, signals=[15]) 2024-08-31T02:10:35.016 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-31T02:10:35.023 INFO:teuthology.task.internal:Opening connections... 2024-08-31T02:10:35.023 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi070.front.sepia.ceph.com 2024-08-31T02:10:35.024 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T02:10:35.105 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi102.front.sepia.ceph.com 2024-08-31T02:10:35.105 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T02:10:35.181 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-31T02:10:35.188 DEBUG:teuthology.orchestra.run.smithi070:> uname -m 2024-08-31T02:10:35.206 INFO:teuthology.orchestra.run.smithi070.stdout:x86_64 2024-08-31T02:10:35.206 DEBUG:teuthology.orchestra.run.smithi070:> cat /etc/os-release 2024-08-31T02:10:35.263 INFO:teuthology.orchestra.run.smithi070.stdout:NAME="CentOS Stream" 2024-08-31T02:10:35.263 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION="9" 2024-08-31T02:10:35.263 INFO:teuthology.orchestra.run.smithi070.stdout:ID="centos" 2024-08-31T02:10:35.263 INFO:teuthology.orchestra.run.smithi070.stdout:ID_LIKE="rhel fedora" 2024-08-31T02:10:35.263 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION_ID="9" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:PLATFORM_ID="platform:el9" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:ANSI_COLOR="0;31" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:LOGO="fedora-logo-icon" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:HOME_URL="https://centos.org/" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-31T02:10:35.264 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-31T02:10:35.265 INFO:teuthology.lock.ops:Updating smithi070.front.sepia.ceph.com on lock server 2024-08-31T02:10:35.287 DEBUG:teuthology.orchestra.run.smithi102:> uname -m 2024-08-31T02:10:35.305 INFO:teuthology.orchestra.run.smithi102.stdout:x86_64 2024-08-31T02:10:35.305 DEBUG:teuthology.orchestra.run.smithi102:> cat /etc/os-release 2024-08-31T02:10:35.362 INFO:teuthology.orchestra.run.smithi102.stdout:NAME="CentOS Stream" 2024-08-31T02:10:35.362 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION="9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:ID="centos" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:ID_LIKE="rhel fedora" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION_ID="9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:PLATFORM_ID="platform:el9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:ANSI_COLOR="0;31" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:LOGO="fedora-logo-icon" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:HOME_URL="https://centos.org/" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-31T02:10:35.363 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-31T02:10:35.364 INFO:teuthology.lock.ops:Updating smithi102.front.sepia.ceph.com on lock server 2024-08-31T02:10:35.387 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-31T02:10:35.395 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-31T02:10:35.400 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-31T02:10:35.401 DEBUG:teuthology.orchestra.run.smithi070:> test '!' -e /home/ubuntu/cephtest 2024-08-31T02:10:35.405 DEBUG:teuthology.orchestra.run.smithi102:> test '!' -e /home/ubuntu/cephtest 2024-08-31T02:10:35.421 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-31T02:10:35.427 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-31T02:10:35.428 DEBUG:teuthology.orchestra.run.smithi070:> test -z $(ls -A /var/lib/ceph) 2024-08-31T02:10:35.461 DEBUG:teuthology.orchestra.run.smithi102:> test -z $(ls -A /var/lib/ceph) 2024-08-31T02:10:35.496 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-31T02:10:35.551 INFO:teuthology.run_tasks:Running task kernel... 2024-08-31T02:10:35.566 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-31T02:10:35.566 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-31T02:10:35.567 DEBUG:teuthology.orchestra.run.smithi070:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-31T02:10:35.567 DEBUG:teuthology.orchestra.run.smithi102:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-31T02:10:35.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:10:35.583 DEBUG:teuthology.orchestra.run.smithi102:> uname -r 2024-08-31T02:10:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:10:35.584 DEBUG:teuthology.orchestra.run.smithi070:> uname -r 2024-08-31T02:10:35.639 INFO:teuthology.orchestra.run.smithi102.stdout:5.14.0-503.el9.x86_64 2024-08-31T02:10:35.640 INFO:teuthology.task.kernel:Running kernel on smithi102: 5.14.0-503.el9.x86_64 2024-08-31T02:10:35.640 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum install -y kernel 2024-08-31T02:10:35.643 INFO:teuthology.orchestra.run.smithi070.stdout:5.14.0-503.el9.x86_64 2024-08-31T02:10:35.643 INFO:teuthology.task.kernel:Running kernel on smithi070: 5.14.0-503.el9.x86_64 2024-08-31T02:10:35.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum install -y kernel 2024-08-31T02:10:39.297 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - BaseOS 4.4 MB/s | 8.2 MB 00:01 2024-08-31T02:10:40.251 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - BaseOS 2.8 MB/s | 8.2 MB 00:02 2024-08-31T02:10:42.127 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - AppStream 26 MB/s | 20 MB 00:00 2024-08-31T02:10:44.006 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - AppStream 5.7 MB/s | 20 MB 00:03 2024-08-31T02:10:46.712 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - CRB 8.3 MB/s | 6.6 MB 00:00 2024-08-31T02:10:48.212 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 9 - Extras packages 60 kB/s | 18 kB 00:00 2024-08-31T02:10:49.125 INFO:teuthology.orchestra.run.smithi102.stdout:Extra Packages for Enterprise Linux 28 MB/s | 22 MB 00:00 2024-08-31T02:10:49.481 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.6 MB 00:01 2024-08-31T02:10:51.015 INFO:teuthology.orchestra.run.smithi070.stdout:CentOS Stream 9 - Extras packages 58 kB/s | 18 kB 00:00 2024-08-31T02:10:51.777 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 35 MB/s | 22 MB 00:00 2024-08-31T02:10:54.573 INFO:teuthology.orchestra.run.smithi102.stdout:lab-extras 44 kB/s | 1.7 kB 00:00 2024-08-31T02:10:56.546 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-31T02:10:56.546 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-31T02:10:56.639 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-08-31T02:10:56.640 INFO:teuthology.orchestra.run.smithi102.stdout:Nothing to do. 2024-08-31T02:10:56.640 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-08-31T02:10:56.737 DEBUG:teuthology.orchestra.run.smithi102:> echo no | sudo yum reinstall kernel || true 2024-08-31T02:10:57.133 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 22 kB/s | 1.7 kB 00:00 2024-08-31T02:10:57.275 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:03 ago on Sat 31 Aug 2024 02:10:54 AM UTC. 2024-08-31T02:10:57.416 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-08-31T02:10:57.424 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:57.424 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-08-31T02:10:57.424 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:57.424 INFO:teuthology.orchestra.run.smithi102.stdout:Reinstalling: 2024-08-31T02:10:57.424 INFO:teuthology.orchestra.run.smithi102.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 1.9 M 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 0 2024-08-31T02:10:57.425 INFO:teuthology.orchestra.run.smithi102.stderr:Operation aborted. 2024-08-31T02:10:57.496 INFO:teuthology.orchestra.run.smithi102.stdout:Is this ok [y/N]: 2024-08-31T02:10:57.496 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum reinstall -y kernel || true 2024-08-31T02:10:58.033 INFO:teuthology.orchestra.run.smithi102.stdout:Last metadata expiration check: 0:00:04 ago on Sat 31 Aug 2024 02:10:54 AM UTC. 2024-08-31T02:10:58.160 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:Reinstalling: 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 1.9 M 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 0 2024-08-31T02:10:58.161 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-08-31T02:10:58.950 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-5.14.0-503.el9.x86_64.rpm 3.5 MB/s | 1.9 MB 00:00 2024-08-31T02:10:58.950 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-08-31T02:10:58.950 INFO:teuthology.orchestra.run.smithi102.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2024-08-31T02:10:58.988 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-08-31T02:10:59.173 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-31T02:10:59.173 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-31T02:10:59.228 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-08-31T02:10:59.228 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-08-31T02:10:59.253 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-08-31T02:10:59.254 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2024-08-31T02:10:59.255 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-08-31T02:10:59.258 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-08-31T02:10:59.258 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-08-31T02:10:59.365 DEBUG:teuthology.orchestra.run.smithi070:> echo no | sudo yum reinstall kernel || true 2024-08-31T02:10:59.437 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-08-31T02:10:59.527 INFO:teuthology.orchestra.run.smithi102.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T02:10:59.527 INFO:teuthology.orchestra.run.smithi102.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:10:59.943 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:02 ago on Sat 31 Aug 2024 02:10:57 AM UTC. 2024-08-31T02:11:00.097 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-08-31T02:11:00.113 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:Reinstalling: 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 1.9 M 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 0 2024-08-31T02:11:00.114 INFO:teuthology.orchestra.run.smithi070.stderr:Operation aborted. 2024-08-31T02:11:00.177 INFO:teuthology.orchestra.run.smithi070.stdout:Is this ok [y/N]: 2024-08-31T02:11:00.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum reinstall -y kernel || true 2024-08-31T02:11:00.248 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:11:00.248 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T02:11:00.713 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:03 ago on Sat 31 Aug 2024 02:10:57 AM UTC. 2024-08-31T02:11:00.839 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2024-08-31T02:11:00.840 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.840 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2024-08-31T02:11:00.840 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.840 INFO:teuthology.orchestra.run.smithi070.stdout:Reinstalling: 2024-08-31T02:11:00.840 INFO:teuthology.orchestra.run.smithi070.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 1.9 M 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 0 2024-08-31T02:11:00.841 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout:Reinstalled: 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:11:00.985 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-08-31T02:11:01.101 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-5.14.0-503.el9.x86_64.rpm 9.1 MB/s | 1.9 MB 00:00 2024-08-31T02:11:01.101 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2024-08-31T02:11:01.102 INFO:teuthology.orchestra.run.smithi070.stdout:Total 7.2 MB/s | 1.9 MB 00:00 2024-08-31T02:11:01.133 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2024-08-31T02:11:01.216 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q kernel | sort -rV | head -n 1 2024-08-31T02:11:01.276 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-31T02:11:01.276 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-31T02:11:01.277 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-31T02:11:01.277 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-31T02:11:01.277 DEBUG:teuthology.orchestra.run.smithi102:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-31T02:11:01.307 INFO:teuthology.orchestra.run.smithi102.stdout:ttyS1 2024-08-31T02:11:01.333 DEBUG:teuthology.parallel:result is None 2024-08-31T02:11:01.363 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2024-08-31T02:11:01.363 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2024-08-31T02:11:01.393 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2024-08-31T02:11:01.394 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2024-08-31T02:11:01.583 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2024-08-31T02:11:01.687 INFO:teuthology.orchestra.run.smithi070.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T02:11:01.687 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:11:02.394 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:11:02.395 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout:Reinstalled: 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:11:03.082 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2024-08-31T02:11:03.275 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q kernel | sort -rV | head -n 1 2024-08-31T02:11:03.333 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-31T02:11:03.334 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-31T02:11:03.334 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-31T02:11:03.334 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-31T02:11:03.334 DEBUG:teuthology.orchestra.run.smithi070:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-31T02:11:03.367 INFO:teuthology.orchestra.run.smithi070.stdout:ttyS1 2024-08-31T02:11:03.402 DEBUG:teuthology.parallel:result is None 2024-08-31T02:11:03.402 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-31T02:11:03.409 INFO:teuthology.task.internal:Creating test directory... 2024-08-31T02:11:03.409 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-31T02:11:03.444 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-31T02:11:03.462 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-31T02:11:03.490 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-31T02:11:03.499 INFO:teuthology.task.internal:Creating archive directory... 2024-08-31T02:11:03.500 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-31T02:11:03.504 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-31T02:11:03.538 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-31T02:11:03.545 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-31T02:11:03.545 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-31T02:11:03.570 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-31T02:11:03.599 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T02:11:03.609 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T02:11:03.638 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T02:11:03.649 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T02:11:03.651 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-31T02:11:03.658 INFO:teuthology.task.internal:Configuring sudo... 2024-08-31T02:11:03.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-31T02:11:03.661 DEBUG:teuthology.orchestra.run.smithi102:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-31T02:11:03.687 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-31T02:11:03.696 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-31T02:11:03.697 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-31T02:11:03.726 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-31T02:11:03.744 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T02:11:03.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T02:11:03.871 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T02:11:03.953 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T02:11:03.993 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:11:03.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-31T02:11:04.057 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T02:11:04.102 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T02:11:04.150 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T02:11:04.231 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T02:11:04.276 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:11:04.276 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-31T02:11:04.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo service rsyslog restart 2024-08-31T02:11:04.345 DEBUG:teuthology.orchestra.run.smithi102:> sudo service rsyslog restart 2024-08-31T02:11:04.377 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T02:11:04.419 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T02:11:04.740 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-31T02:11:04.750 INFO:teuthology.task.internal:Starting timer... 2024-08-31T02:11:04.751 INFO:teuthology.run_tasks:Running task pcp... 2024-08-31T02:11:04.765 INFO:teuthology.run_tasks:Running task selinux... 2024-08-31T02:11:04.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo service auditd rotate 2024-08-31T02:11:04.818 INFO:teuthology.orchestra.run.smithi070.stdout:Rotating logs: 2024-08-31T02:11:04.820 DEBUG:teuthology.orchestra.run.smithi102:> sudo service auditd rotate 2024-08-31T02:11:04.875 INFO:teuthology.orchestra.run.smithi102.stdout:Rotating logs: 2024-08-31T02:11:04.877 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-31T02:11:04.877 DEBUG:teuthology.orchestra.run.smithi070:> /usr/sbin/getenforce 2024-08-31T02:11:04.897 INFO:teuthology.orchestra.run.smithi070.stdout:Permissive 2024-08-31T02:11:04.898 DEBUG:teuthology.orchestra.run.smithi102:> /usr/sbin/getenforce 2024-08-31T02:11:04.918 INFO:teuthology.orchestra.run.smithi102.stdout:Permissive 2024-08-31T02:11:04.918 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi070.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi102.front.sepia.ceph.com': 'permissive'} 2024-08-31T02:11:04.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T02:11:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:11:04.964 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T02:11:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:11:04.991 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-31T02:11:04.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo /usr/sbin/setenforce permissive 2024-08-31T02:11:05.034 DEBUG:teuthology.orchestra.run.smithi102:> sudo /usr/sbin/setenforce permissive 2024-08-31T02:11:05.067 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-31T02:11:05.077 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-08-31T02:11:05.080 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-08-31T02:11:05.198 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-31T02:11:05.209 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-31T02:11:05.210 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi070.front.sepia.ceph.com,smithi102.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-31T02:12:50.221 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi070.front.sepia.ceph.com'), Remote(name='ubuntu@smithi102.front.sepia.ceph.com')] 2024-08-31T02:12:50.223 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-08-31T02:12:50.224 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T02:12:50.302 DEBUG:teuthology.orchestra.run.smithi070:> true 2024-08-31T02:12:50.385 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi070.front.sepia.ceph.com' 2024-08-31T02:12:50.385 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-08-31T02:12:50.386 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T02:12:50.459 DEBUG:teuthology.orchestra.run.smithi102:> true 2024-08-31T02:12:50.536 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-08-31T02:12:50.536 INFO:teuthology.run_tasks:Running task clock... 2024-08-31T02:12:50.545 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-31T02:12:50.545 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-31T02:12:50.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T02:12:50.548 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-31T02:12:50.548 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T02:12:50.577 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-31T02:12:50.592 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-31T02:12:50.605 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-31T02:12:50.617 INFO:teuthology.orchestra.run.smithi070.stderr:sudo: ntpd: command not found 2024-08-31T02:12:50.619 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-31T02:12:50.628 INFO:teuthology.orchestra.run.smithi070.stdout:506 Cannot talk to daemon 2024-08-31T02:12:50.642 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-31T02:12:50.643 INFO:teuthology.orchestra.run.smithi102.stderr:sudo: ntpd: command not found 2024-08-31T02:12:50.654 INFO:teuthology.orchestra.run.smithi102.stdout:506 Cannot talk to daemon 2024-08-31T02:12:50.656 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-31T02:12:50.669 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-31T02:12:50.683 INFO:teuthology.orchestra.run.smithi102.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-31T02:12:50.713 INFO:teuthology.orchestra.run.smithi070.stderr:bash: line 1: ntpq: command not found 2024-08-31T02:12:50.717 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T02:12:50.717 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2024-08-31T02:12:50.717 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.717 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.718 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.718 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.745 INFO:teuthology.orchestra.run.smithi102.stderr:bash: line 1: ntpq: command not found 2024-08-31T02:12:50.747 INFO:teuthology.orchestra.run.smithi102.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T02:12:50.748 INFO:teuthology.orchestra.run.smithi102.stdout:=============================================================================== 2024-08-31T02:12:50.748 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.748 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.748 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.748 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:12:50.749 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-31T02:12:50.837 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': True}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': '34b9d27bc925f169ace27b9f1f204536aa49fc0b'} 2024-08-31T02:12:50.837 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-08-31T02:12:50.837 INFO:tasks.cephadm:Cluster fsid is 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:12:50.837 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-31T02:12:50.838 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.70', 'mon.c': '[v2:172.21.15.70:3301,v1:172.21.15.70:6790]', 'mon.b': '172.21.15.102'} 2024-08-31T02:12:50.838 INFO:tasks.cephadm:First mon is mon.a on smithi070 2024-08-31T02:12:50.838 INFO:tasks.cephadm:First mgr is y 2024-08-31T02:12:50.838 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-31T02:12:50.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo hostname $(hostname -s) 2024-08-31T02:12:50.864 DEBUG:teuthology.orchestra.run.smithi102:> sudo hostname $(hostname -s) 2024-08-31T02:12:50.889 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-08-31T02:12:50.890 DEBUG:teuthology.orchestra.run.smithi070:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.024 INFO:teuthology.orchestra.run.smithi070.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Aug 31 02:12 /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.025 DEBUG:teuthology.orchestra.run.smithi102:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.149 INFO:teuthology.orchestra.run.smithi102.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Aug 31 02:12 /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.149 DEBUG:teuthology.orchestra.run.smithi070:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.166 DEBUG:teuthology.orchestra.run.smithi102:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-31T02:12:51.192 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-08-31T02:12:51.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-08-31T02:12:51.209 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-08-31T02:12:52.739 INFO:teuthology.orchestra.run.smithi070.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-08-31T02:12:52.891 INFO:teuthology.orchestra.run.smithi102.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-08-31T02:13:04.072 INFO:teuthology.orchestra.run.smithi102.stdout:{ 2024-08-31T02:13:04.072 INFO:teuthology.orchestra.run.smithi102.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout: "repo_digests": [ 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout: ] 2024-08-31T02:13:04.073 INFO:teuthology.orchestra.run.smithi102.stdout:} 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "repo_digests": [ 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2024-08-31T02:13:04.508 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:13:04.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo mkdir -p /etc/ceph 2024-08-31T02:13:04.588 DEBUG:teuthology.orchestra.run.smithi102:> sudo mkdir -p /etc/ceph 2024-08-31T02:13:04.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 777 /etc/ceph 2024-08-31T02:13:04.642 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 777 /etc/ceph 2024-08-31T02:13:04.681 INFO:tasks.cephadm:Writing seed config... 2024-08-31T02:13:04.682 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = True 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-31T02:13:04.683 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-31T02:13:04.684 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:13:04.684 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-31T02:13:04.699 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 862b0c56-673e-11ef-bcd4-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = True [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-31T02:13:04.700 DEBUG:teuthology.orchestra.run.smithi070:mon.a> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service 2024-08-31T02:13:04.742 DEBUG:teuthology.orchestra.run.smithi070:mgr.y> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service 2024-08-31T02:13:04.784 INFO:tasks.cephadm:Bootstrapping... 2024-08-31T02:13:04.785 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.70 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:13:04.972 INFO:teuthology.orchestra.run.smithi070.stderr:-------------------------------------------------------------------------------- 2024-08-31T02:13:04.972 INFO:teuthology.orchestra.run.smithi070.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '862b0c56-673e-11ef-bcd4-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.70', '--skip-admin-label'] 2024-08-31T02:13:04.995 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: 5.2.2 2024-08-31T02:13:04.997 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying podman|docker is present... 2024-08-31T02:13:05.018 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: 5.2.2 2024-08-31T02:13:05.020 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying lvm2 is present... 2024-08-31T02:13:05.020 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying time synchronization is in place... 2024-08-31T02:13:05.027 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-08-31T02:13:05.034 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:13:05.040 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: enabled 2024-08-31T02:13:05.047 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: active 2024-08-31T02:13:05.047 INFO:teuthology.orchestra.run.smithi070.stderr:Unit chronyd.service is enabled and running 2024-08-31T02:13:05.047 INFO:teuthology.orchestra.run.smithi070.stderr:Repeating the final host check... 2024-08-31T02:13:05.067 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: 5.2.2 2024-08-31T02:13:05.069 INFO:teuthology.orchestra.run.smithi070.stderr:podman (/bin/podman) version 5.2.2 is present 2024-08-31T02:13:05.069 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl is present 2024-08-31T02:13:05.069 INFO:teuthology.orchestra.run.smithi070.stderr:lvcreate is present 2024-08-31T02:13:05.075 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-08-31T02:13:05.082 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:13:05.089 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: enabled 2024-08-31T02:13:05.095 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: active 2024-08-31T02:13:05.095 INFO:teuthology.orchestra.run.smithi070.stderr:Unit chronyd.service is enabled and running 2024-08-31T02:13:05.096 INFO:teuthology.orchestra.run.smithi070.stderr:Host looks OK 2024-08-31T02:13:05.096 INFO:teuthology.orchestra.run.smithi070.stderr:Cluster fsid: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:05.096 INFO:teuthology.orchestra.run.smithi070.stderr:Acquiring lock 139722813543040 on /run/cephadm/862b0c56-673e-11ef-bcd4-c7b262605968.lock 2024-08-31T02:13:05.096 INFO:teuthology.orchestra.run.smithi070.stderr:Lock 139722813543040 acquired on /run/cephadm/862b0c56-673e-11ef-bcd4-c7b262605968.lock 2024-08-31T02:13:05.096 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying IP 172.21.15.70 port 3300 ... 2024-08-31T02:13:05.097 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying IP 172.21.15.70 port 6789 ... 2024-08-31T02:13:05.097 INFO:teuthology.orchestra.run.smithi070.stderr:Base mon IP is 172.21.15.70, final addrv is [v2:172.21.15.70:3300,v1:172.21.15.70:6789] 2024-08-31T02:13:05.099 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.70 metric 100 2024-08-31T02:13:05.099 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.70 metric 100 2024-08-31T02:13:05.101 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-08-31T02:13:05.102 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-08-31T02:13:05.102 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-08-31T02:13:05.104 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-31T02:13:05.104 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-08-31T02:13:05.104 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-08-31T02:13:05.105 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-08-31T02:13:05.105 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d3e7/64 scope link noprefixroute 2024-08-31T02:13:05.105 INFO:teuthology.orchestra.run.smithi070.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-08-31T02:13:05.105 INFO:teuthology.orchestra.run.smithi070.stderr:Mon IP `172.21.15.70` is in CIDR network `172.21.0.0/20` 2024-08-31T02:13:05.105 INFO:teuthology.orchestra.run.smithi070.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-31T02:13:05.106 INFO:teuthology.orchestra.run.smithi070.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-08-31T02:13:05.205 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Getting image source signatures 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-08-31T02:13:05.537 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-08-31T02:13:05.711 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: Writing manifest to image destination 2024-08-31T02:13:05.855 INFO:teuthology.orchestra.run.smithi070.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-08-31T02:13:06.234 INFO:teuthology.orchestra.run.smithi070.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-08-31T02:13:06.779 INFO:teuthology.orchestra.run.smithi070.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-08-31T02:13:06.779 INFO:teuthology.orchestra.run.smithi070.stderr:Extracting ceph user uid/gid from container image... 2024-08-31T02:13:07.179 INFO:teuthology.orchestra.run.smithi070.stderr:stat: 167 167 2024-08-31T02:13:07.626 INFO:teuthology.orchestra.run.smithi070.stderr:Creating initial keys... 2024-08-31T02:13:08.035 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph-authtool: AQAzfNJmXvqWOBAA4ZTtdZFvJ+II3DNU8bXDhQ== 2024-08-31T02:13:14.759 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph-authtool: AQA6fNJmZnLFBRAAwwiWxw/lZQnWPkqKt1mKyA== 2024-08-31T02:13:20.998 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph-authtool: AQBAfNJmdJB3NhAA+iBRrSCd7UvjmIN9BO25eg== 2024-08-31T02:13:21.506 INFO:teuthology.orchestra.run.smithi070.stderr:Creating initial monmap... 2024-08-31T02:13:21.914 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-31T02:13:21.914 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-08-31T02:13:21.914 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:21.914 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr:monmaptool for a [v2:172.21.15.70:3300,v1:172.21.15.70:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr:setting min_mon_release = octopus 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: set fsid to 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:13:22.378 INFO:teuthology.orchestra.run.smithi070.stderr:Creating mon... 2024-08-31T02:13:23.334 INFO:teuthology.orchestra.run.smithi070.stderr:create mon.a on 2024-08-31T02:13:23.546 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-31T02:13:23.702 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target → /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target. 2024-08-31T02:13:23.702 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target → /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target. 2024-08-31T02:13:24.036 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Failed to reset failed state of unit ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service: Unit ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service not loaded. 2024-08-31T02:13:24.044 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Created symlink /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target.wants/ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service → /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968@.service. 2024-08-31T02:13:24.751 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: disabled 2024-08-31T02:13:24.758 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:13:24.758 INFO:teuthology.orchestra.run.smithi070.stderr:firewalld.service is not enabled 2024-08-31T02:13:24.758 INFO:teuthology.orchestra.run.smithi070.stderr:Not possible to enable service . firewalld.service is not available 2024-08-31T02:13:24.759 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mon to start... 2024-08-31T02:13:24.759 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mon... 2024-08-31T02:13:25.287 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: cluster: 2024-08-31T02:13:25.287 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: id: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:25.287 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-08-31T02:13:25.287 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:25.287 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: services: 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.591873s) 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mgr: no daemons active 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: data: 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: pgs: 2024-08-31T02:13:25.288 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:25.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:25 smithi070 ceph-mon[11866]: from='client.? 172.21.15.70:0/2356341257' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:13:26.031 INFO:teuthology.orchestra.run.smithi070.stderr:mon is available 2024-08-31T02:13:26.031 INFO:teuthology.orchestra.run.smithi070.stderr:Assimilating anything we can from ceph.conf... 2024-08-31T02:13:26.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:26.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [global] 2024-08-31T02:13:26.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: fsid = 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.70:3300,v1:172.21.15.70:6789] 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [mgr] 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = True 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [osd] 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-08-31T02:13:26.578 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-08-31T02:13:26.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:26 smithi070 ceph-mon[11866]: from='client.? 172.21.15.70:0/3628744443' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-31T02:13:26.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:26 smithi070 ceph-mon[11866]: from='client.? 172.21.15.70:0/3628744443' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-31T02:13:27.177 INFO:teuthology.orchestra.run.smithi070.stderr:Generating new minimal ceph.conf... 2024-08-31T02:13:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:27 smithi070 ceph-mon[11866]: from='client.? 172.21.15.70:0/267530533' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:13:28.201 INFO:teuthology.orchestra.run.smithi070.stderr:Restarting the monitor... 2024-08-31T02:13:28.562 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 systemd[1]: Stopping Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:13:28.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 bash[12083]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon.a" found: no such container 2024-08-31T02:13:28.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[11862]: 2024-08-31T02:13:28.367+0000 7f3cfa919700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:13:28.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[11862]: 2024-08-31T02:13:28.367+0000 7f3cfa919700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-31T02:13:28.827 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 podman[12093]: 2024-08-31 02:13:28.562073319 +0000 UTC m=+0.296048540 container died d8c3220b3aeb746dcc495f467c49f4f4febe64e8203d9f8506c55e1ea296c837 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870) 2024-08-31T02:13:29.177 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 podman[12093]: 2024-08-31 02:13:28.91757221 +0000 UTC m=+0.651547428 container cleanup d8c3220b3aeb746dcc495f467c49f4f4febe64e8203d9f8506c55e1ea296c837 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, version=8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-type=git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, RELEASE=HEAD, release=754, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:13:29.177 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 bash[12093]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a 2024-08-31T02:13:29.177 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:28 smithi070 bash[12121]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon.a" found: no such container 2024-08-31T02:13:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 podman[12105]: 2024-08-31 02:13:29.176624367 +0000 UTC m=+0.612002670 container remove d8c3220b3aeb746dcc495f467c49f4f4febe64e8203d9f8506c55e1ea296c837 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, version=8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_CLEAN=True, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:13:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-31T02:13:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 systemd[1]: Stopped Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:13:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 systemd[1]: Starting Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:13:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 podman[12182]: 2024-08-31 02:13:29.317024364 +0000 UTC m=+0.016716476 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-08-31T02:13:29.939 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 podman[12182]: 2024-08-31 02:13:29.635089556 +0000 UTC m=+0.334781651 container create 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, release=754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-08-31T02:13:30.025 INFO:teuthology.orchestra.run.smithi070.stderr:Setting mon public_network to 172.21.0.0/20 2024-08-31T02:13:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 podman[12182]: 2024-08-31 02:13:29.939023675 +0000 UTC m=+0.638715780 container init 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.buildah.version=1.19.8, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-08-31T02:13:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 podman[12182]: 2024-08-31 02:13:29.943673301 +0000 UTC m=+0.643365397 container start 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, vcs-type=git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8) 2024-08-31T02:13:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: set uid:gid to 167:167 (ceph:ceph) 2024-08-31T02:13:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-08-31T02:13:30.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: pidfile_write: ignore empty --pid-file 2024-08-31T02:13:30.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: load: jerasure load: lrc 2024-08-31T02:13:30.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: RocksDB version: 6.15.5 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Compile date Apr 18 2022 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: DB SUMMARY 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: DB Session ID: 2UZENG35BK5LN09DHU3P 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: CURRENT file: CURRENT 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: IDENTITY file: IDENTITY 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82633 ; 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.error_if_exists: 0 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.create_if_missing: 0 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.paranoid_checks: 1 2024-08-31T02:13:30.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.env: 0x5570954c6860 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.fs: Posix File System 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.info_log: 0x557096bc1dc0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_file_opening_threads: 16 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.statistics: (nil) 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.use_fsync: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_log_file_size: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.keep_log_file_num: 1000 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.recycle_log_file_num: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_fallocate: 1 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_mmap_reads: 0 2024-08-31T02:13:30.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_mmap_writes: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.use_direct_reads: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.create_missing_column_families: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.db_log_dir: 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.advise_random_on_open: 1 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.db_write_buffer_size: 0 2024-08-31T02:13:30.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.write_buffer_manager: 0x557096cb22a0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.rate_limiter: (nil) 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.wal_recovery_mode: 2 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.enable_thread_tracking: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.enable_pipelined_write: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.unordered_write: 0 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-31T02:13:30.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.row_cache: None 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.wal_filter: None 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_ingest_behind: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.preserve_deletes: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.two_write_queues: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.manual_wal_flush: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.atomic_flush: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.log_readahead_size: 0 2024-08-31T02:13:30.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.best_efforts_recovery: 0 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.allow_data_in_errors: 0 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.db_host_id: __hostname__ 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_background_jobs: 2 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_background_compactions: -1 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_subcompactions: 1 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_total_wal_size: 0 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-31T02:13:30.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_open_files: -1 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bytes_per_sync: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_readahead_size: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_background_flushes: -1 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Compression algorithms supported: 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kZSTD supported: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kXpressCompression supported: 0 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kLZ4HCCompression supported: 1 2024-08-31T02:13:30.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kLZ4Compression supported: 1 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kBZip2Compression supported: 0 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kZlibCompression supported: 1 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: kSnappyCompression supported: 1 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.merge_operator: 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_filter: None 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_filter_factory: None 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.sst_partitioner_factory: None 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-31T02:13:30.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557096b8dd00) 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_type: 0 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_index_type: 0 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_shortening: 1 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: hash_index_allow_collision: 1 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: checksum: 1 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: no_block_cache: 0 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache: 0x557096bf8170 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_options: 2024-08-31T02:13:30.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: capacity : 536870912 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_shard_bits : 4 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: strict_capacity_limit : 0 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_compressed: (nil) 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: persistent_cache: (nil) 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size: 4096 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size_deviation: 10 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_restart_interval: 16 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_block_restart_interval: 1 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: metadata_block_size: 4096 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: partition_filters: 0 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: use_delta_encoding: 1 2024-08-31T02:13:30.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: whole_key_filtering: 1 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: verify_compression: 0 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: format_version: 4 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_index_compression: 1 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_align: 0 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.write_buffer_size: 33554432 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_write_buffer_number: 2 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression: NoCompression 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression: Disabled 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.prefix_extractor: nullptr 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.num_levels: 7 2024-08-31T02:13:30.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.level: 32767 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.strategy: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-31T02:13:30.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:13:30.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-31T02:13:30.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compression_opts.enabled: false 2024-08-31T02:13:30.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.target_file_size_base: 67108864 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-31T02:13:30.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.arena_block_size: 4194304 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.disable_auto_compactions: 0 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-31T02:13:30.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.table_properties_collectors: 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.inplace_update_support: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.bloom_locality: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.max_successive_merges: 0 2024-08-31T02:13:30.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.paranoid_file_checks: 0 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.force_consistency_checks: 1 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.report_bg_io_stats: 0 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.ttl: 2592000 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.enable_blob_files: false 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.min_blob_size: 0 2024-08-31T02:13:30.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.blob_file_size: 268435456 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725070409972194, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725070409972953, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79183, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77446, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71755, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1725070409, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "3f9bc000-d3bb-42c9-aad3-42ec4ce98def", "db_session_id": "2UZENG35BK5LN09DHU3P"}} 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725070409973306, "job": 1, "event": "recovery_finished"} 2024-08-31T02:13:30.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x557096bdea80 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: DB pointer 0x557096bee000 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** DB Stats ** 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.219 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-31T02:13:30.220 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 22.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 22.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:13:30.221 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 145.5 0.00 0.00 1 0.001 0 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:13:30.222 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 22.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: starting mon.a rank 0 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???) e1 preinit fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).mds e1 new map 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).mds e1 print_map 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: e1 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-31T02:13:30.223 INFO:journalctl@ceph.mon.a.smithi070.stdout: legacy client fscid: -1 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout: No filesystems configured 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expand map: {default=false} 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta from 'false' to 'false' 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expanded map: {default=false} 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expand map: {default=info} 2024-08-31T02:13:30.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta from 'info' to 'info' 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expanded map: {default=info} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expand map: {default=daemon} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta from 'daemon' to 'daemon' 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expanded map: {default=daemon} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expand map: {default=debug} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta from 'debug' to 'debug' 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:29 smithi070 ceph-mon[12201]: expand_channel_meta expanded map: {default=debug} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 bash[12182]: 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 systemd[1]: Started Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 ceph-mon[12201]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 ceph-mon[12201]: monmap e1: 1 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 ceph-mon[12201]: osdmap e1: 0 total, 0 up, 0 in 2024-08-31T02:13:30.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:30 smithi070 ceph-mon[12201]: mgrmap e1: no daemons active 2024-08-31T02:13:31.073 INFO:teuthology.orchestra.run.smithi070.stderr:Wrote config to /etc/ceph/ceph.conf 2024-08-31T02:13:31.073 INFO:teuthology.orchestra.run.smithi070.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:13:31.073 INFO:teuthology.orchestra.run.smithi070.stderr:Creating mgr... 2024-08-31T02:13:31.074 INFO:teuthology.orchestra.run.smithi070.stderr:Verifying port 9283 ... 2024-08-31T02:13:31.247 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Failed to reset failed state of unit ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service: Unit ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service not loaded. 2024-08-31T02:13:31.254 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: Created symlink /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968.target.wants/ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-862b0c56-673e-11ef-bcd4-c7b262605968@.service. 2024-08-31T02:13:31.423 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:31 smithi070 systemd[1]: Starting Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:13:31.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3148196094' entity='client.admin' 2024-08-31T02:13:31.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:31 smithi070 podman[12413]: 2024-08-31 02:13:31.572997241 +0000 UTC m=+0.015955371 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-08-31T02:13:32.249 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:31 smithi070 podman[12413]: 2024-08-31 02:13:31.963240572 +0000 UTC m=+0.406198744 container create 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, distribution-scope=public, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container) 2024-08-31T02:13:32.249 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 podman[12413]: 2024-08-31 02:13:32.165449648 +0000 UTC m=+0.608407772 container init 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, release=754, GIT_CLEAN=True) 2024-08-31T02:13:32.249 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 podman[12413]: 2024-08-31 02:13:32.169129064 +0000 UTC m=+0.612087190 container start 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_CLEAN=True, ceph=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:13:32.249 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 bash[12413]: 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e 2024-08-31T02:13:32.264 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: disabled 2024-08-31T02:13:32.272 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:13:32.273 INFO:teuthology.orchestra.run.smithi070.stderr:firewalld.service is not enabled 2024-08-31T02:13:32.273 INFO:teuthology.orchestra.run.smithi070.stderr:Not possible to enable service . firewalld.service is not available 2024-08-31T02:13:32.283 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: disabled 2024-08-31T02:13:32.291 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:13:32.292 INFO:teuthology.orchestra.run.smithi070.stderr:firewalld.service is not enabled 2024-08-31T02:13:32.292 INFO:teuthology.orchestra.run.smithi070.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-08-31T02:13:32.292 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mgr to start... 2024-08-31T02:13:32.292 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mgr... 2024-08-31T02:13:32.540 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 systemd[1]: Started Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:13:32.541 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:32.323+0000 7fc3be351000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:13:32.541 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:32.401+0000 7fc3be351000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:13:32.805 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:32.580+0000 7fc3be351000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:13:32.805 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:32.668+0000 7fc3be351000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:13:32.805 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:32.804+0000 7fc3be351000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsid": "862b0c56-673e-11ef-bcd4-c7b262605968", 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "health": { 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "checks": {}, 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mutes": [] 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum": [ 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 0 2024-08-31T02:13:32.868 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_names": [ 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "a" 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "monmap": { 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_mons": 1 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osdmap": { 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_osds": 0, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-08-31T02:13:32.869 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgmap": { 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pools": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_objects": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-08-31T02:13:32.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_total": 0 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsmap": { 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "by_rank": [], 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "up:standby": 0 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap": { 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": false, 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modules": [ 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "iostat", 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "nfs", 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "restful" 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:32.871 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "servicemap": { 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modified": "2024-08-31T02:13:24.693806+0000", 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "progress_events": {} 2024-08-31T02:13:32.872 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:33.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:32 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/642626994' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-31T02:13:33.147 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:33.020+0000 7fc3be351000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:13:33.441 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:33.353+0000 7fc3be351000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:13:33.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:33.440+0000 7fc3be351000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:13:33.458 INFO:teuthology.orchestra.run.smithi070.stderr:mgr not available, waiting (1/15)... 2024-08-31T02:13:33.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:33.603+0000 7fc3be351000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:13:34.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:34.650+0000 7fc3be351000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:13:34.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:34.738+0000 7fc3be351000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:13:35.851 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:35.643+0000 7fc3be351000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsid": "862b0c56-673e-11ef-bcd4-c7b262605968", 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "health": { 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "checks": {}, 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mutes": [] 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum": [ 2024-08-31T02:13:36.010 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 0 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_names": [ 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "a" 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "monmap": { 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_mons": 1 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osdmap": { 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_osds": 0, 2024-08-31T02:13:36.011 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-08-31T02:13:36.012 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-08-31T02:13:36.012 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-08-31T02:13:36.015 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-08-31T02:13:36.015 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgmap": { 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pools": 0, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_objects": 0, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-08-31T02:13:36.016 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_total": 0 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsmap": { 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "by_rank": [], 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "up:standby": 0 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap": { 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": false, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modules": [ 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "iostat", 2024-08-31T02:13:36.017 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "nfs", 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "restful" 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "servicemap": { 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modified": "2024-08-31T02:13:24.693806+0000", 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "progress_events": {} 2024-08-31T02:13:36.018 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:36.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1682057302' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-31T02:13:36.103 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:35.957+0000 7fc3be351000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:13:36.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.100+0000 7fc3be351000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:13:36.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.192+0000 7fc3be351000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:13:36.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.298+0000 7fc3be351000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:13:36.643 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.390+0000 7fc3be351000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:13:36.643 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.546+0000 7fc3be351000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:13:36.643 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.641+0000 7fc3be351000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:13:36.666 INFO:teuthology.orchestra.run.smithi070.stderr:mgr not available, waiting (2/15)... 2024-08-31T02:13:36.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:36.740+0000 7fc3be351000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:13:37.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:37.030+0000 7fc3be351000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:13:37.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:37.120+0000 7fc3be351000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:13:37.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:37.208+0000 7fc3be351000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:13:38.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:38.257+0000 7fc3be351000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:13:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: Activating manager daemon y 2024-08-31T02:13:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: mgrmap e2: y(active, starting, since 0.00282923s) 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' 2024-08-31T02:13:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:38 smithi070 ceph-mon[12201]: from='mgr.14100 172.21.15.70:0/2433229420' entity='mgr.y' 2024-08-31T02:13:39.154 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:39.154 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:39.154 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsid": "862b0c56-673e-11ef-bcd4-c7b262605968", 2024-08-31T02:13:39.154 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "health": { 2024-08-31T02:13:39.154 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "checks": {}, 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mutes": [] 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum": [ 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 0 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_names": [ 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "a" 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "monmap": { 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:39.155 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_mons": 1 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osdmap": { 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_osds": 0, 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-08-31T02:13:39.156 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-08-31T02:13:39.159 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-08-31T02:13:39.159 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgmap": { 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pools": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_objects": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_total": 0 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.160 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsmap": { 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "by_rank": [], 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "up:standby": 0 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap": { 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": false, 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modules": [ 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "iostat", 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "nfs", 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "restful" 2024-08-31T02:13:39.161 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "servicemap": { 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modified": "2024-08-31T02:13:24.693806+0000", 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "progress_events": {} 2024-08-31T02:13:39.162 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:39 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3318573469' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-31T02:13:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:39 smithi070 ceph-mon[12201]: mgrmap e3: y(active, since 1.00586s) 2024-08-31T02:13:39.713 INFO:teuthology.orchestra.run.smithi070.stderr:mgr not available, waiting (3/15)... 2024-08-31T02:13:42.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:41 smithi070 ceph-mon[12201]: mgrmap e4: y(active, since 2s) 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsid": "862b0c56-673e-11ef-bcd4-c7b262605968", 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "health": { 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "checks": {}, 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mutes": [] 2024-08-31T02:13:42.364 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum": [ 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 0 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_names": [ 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "a" 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "monmap": { 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_mons": 1 2024-08-31T02:13:42.365 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osdmap": { 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_osds": 0, 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-08-31T02:13:42.366 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgmap": { 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_pools": 0, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_objects": 0, 2024-08-31T02:13:42.370 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "bytes_total": 0 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "fsmap": { 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "by_rank": [], 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "up:standby": 0 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap": { 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": true, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modules": [ 2024-08-31T02:13:42.371 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "iostat", 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "nfs", 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "restful" 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ], 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "servicemap": { 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 1, 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "modified": "2024-08-31T02:13:24.693806+0000", 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "services": {} 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: }, 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "progress_events": {} 2024-08-31T02:13:42.372 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:42.952 INFO:teuthology.orchestra.run.smithi070.stderr:mgr is available 2024-08-31T02:13:43.170 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:42 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1099965518' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-31T02:13:43.576 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:43.576 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [global] 2024-08-31T02:13:43.576 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: fsid = 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [mgr] 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: [osd] 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-08-31T02:13:43.577 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-08-31T02:13:44.165 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3404222481' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-31T02:13:44.165 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3404222481' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-31T02:13:44.167 INFO:teuthology.orchestra.run.smithi070.stderr:Enabling cephadm module... 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:44 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1870948375' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setuser ceph since I am not root 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setgroup ceph since I am not root 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.144+0000 7f6bbfad8000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.223+0000 7f6bbfad8000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:13:45.398 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.397+0000 7f6bbfad8000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:13:45.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.485+0000 7f6bbfad8000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:13:45.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.616+0000 7f6bbfad8000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:13:45.984 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:45.835+0000 7f6bbfad8000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:13:46.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1870948375' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-31T02:13:46.273 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:45 smithi070 ceph-mon[12201]: mgrmap e5: y(active, since 6s) 2024-08-31T02:13:46.274 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:46.171+0000 7f6bbfad8000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:13:46.274 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:46.272+0000 7f6bbfad8000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 5, 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": true, 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "active_name": "y", 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standby": 0 2024-08-31T02:13:46.412 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:46.542 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:46.443+0000 7f6bbfad8000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:13:46.997 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for the mgr to restart... 2024-08-31T02:13:46.997 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mgr epoch 5... 2024-08-31T02:13:47.316 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:46 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2167273530' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-31T02:13:47.607 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:47.516+0000 7f6bbfad8000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:13:47.607 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:47.606+0000 7f6bbfad8000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:13:48.803 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:48.495+0000 7f6bbfad8000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:13:49.122 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:48.802+0000 7f6bbfad8000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:13:49.122 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:48.935+0000 7f6bbfad8000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:13:49.122 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.023+0000 7f6bbfad8000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:13:49.122 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.121+0000 7f6bbfad8000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:13:49.448 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.209+0000 7f6bbfad8000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:13:49.448 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.354+0000 7f6bbfad8000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:13:49.448 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.447+0000 7f6bbfad8000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:13:49.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.543+0000 7f6bbfad8000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:13:50.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.823+0000 7f6bbfad8000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:13:50.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:49.912+0000 7f6bbfad8000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:13:50.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:50.001+0000 7f6bbfad8000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:13:51.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:13:51.060+0000 7f6bbfad8000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:13:52.124 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:13:52.125 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-08-31T02:13:52.125 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "initialized": true 2024-08-31T02:13:52.125 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:13:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: Active manager daemon y restarted 2024-08-31T02:13:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: Activating manager daemon y 2024-08-31T02:13:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: osdmap e2: 0 total, 0 up, 0 in 2024-08-31T02:13:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: mgrmap e6: y(active, starting, since 0.0526119s) 2024-08-31T02:13:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:13:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:13:52.712 INFO:teuthology.orchestra.run.smithi070.stderr:mgr epoch 5 is available 2024-08-31T02:13:52.712 INFO:teuthology.orchestra.run.smithi070.stderr:Setting orchestrator backend to cephadm... 2024-08-31T02:13:53.053 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:13:52] ENGINE Bus STARTING 2024-08-31T02:13:53.053 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:13:52] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:13:53.053 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:13:52] ENGINE Bus STARTED 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: mgrmap e7: y(active, since 1.05488s) 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: [31/Aug/2024:02:13:52] ENGINE Bus STARTING 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: [31/Aug/2024:02:13:52] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: [31/Aug/2024:02:13:52] ENGINE Bus STARTED 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:53.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:53 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:13:54.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:54 smithi070 ceph-mon[12201]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:13:54.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:54 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:54.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:54 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:13:54.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:54 smithi070 ceph-mon[12201]: mgrmap e8: y(active, since 2s) 2024-08-31T02:13:54.550 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: value unchanged 2024-08-31T02:13:55.124 INFO:teuthology.orchestra.run.smithi070.stderr:Generating ssh key... 2024-08-31T02:13:56.111 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-mon[12201]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:13:56.111 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:56.111 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: Generating public/private rsa key pair. 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: Your identification has been saved in /tmp/tmpypza4s7q/key. 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: Your public key has been saved in /tmp/tmpypza4s7q/key.pub. 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: The key fingerprint is: 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: SHA256:Sk13XL3B6FZf15EgPDuhZCEnAljB/cmWp7Z6rPbnYew ceph-862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: The key's randomart image is: 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: +---[RSA 3072]----+ 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | +++. o oo. .=o+| 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | . . .. +o =.o ==| 2024-08-31T02:13:56.112 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | o * o B . *| 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | O + + o ..| 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | o S o | 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | . +. | 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | + .+ | 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | . +o.. | 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: | .o=.oE | 2024-08-31T02:13:56.113 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:13:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: +----[SHA256]-----+ 2024-08-31T02:13:56.952 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwKb0OqbJ9lGiGbV+0lye719q5g+QoJ0uJxheeFeXBbjcOXAtTwT5unNZIQIiRmI8IOjMZzRawAaXGUIi+AM7xj8w7RZU6AUnXTIuCFAFmUD9JtseUgc5dX9NSEFZKh2Xwxu3PLTj32HamRavOhP1GrayRJ8ygZsdOeghXd8O3D28cJKSoZW9ncRB8lbOdji3n35qIAKQ5mnW8W6ukW0cCWetYN9i/W8nXsw/ZwLBQflW3EwZiDecSMCxuKtlZyNkGWx4araTWFWLNj4WFTZ40TodPg6Fly5Jyys0xrrXfQuCEJ4yVUXvKm8GWnys01EbrVyOYv0XVldIvGsLFO54XTrrToRmhfQeSsp9uFg7seIgsX4UMHd0hTkLVoVT+xsWblYWGzkLIybnXJ19jKJvuyQ7o5/6T5kTlvL/qBVGQwWIB9Xyzgj1X5Ohn9G3F/cISHw8Cm2YCbsDQTCKi0bRYA+aoGnd9Rpa03p4kpC9K85tN8qu3NhzerO6sadY4pys= ceph-862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:13:57.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:56 smithi070 ceph-mon[12201]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:13:57.292 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:56 smithi070 ceph-mon[12201]: Generating ssh key... 2024-08-31T02:13:57.625 INFO:teuthology.orchestra.run.smithi070.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-31T02:13:57.626 INFO:teuthology.orchestra.run.smithi070.stderr:Adding key to root@localhost authorized_keys... 2024-08-31T02:13:57.626 INFO:teuthology.orchestra.run.smithi070.stderr:Adding host smithi070... 2024-08-31T02:13:58.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:57 smithi070 ceph-mon[12201]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:13:59.076 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: Added host 'smithi070' with addr '172.21.15.70' 2024-08-31T02:13:59.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:59 smithi070 ceph-mon[12201]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi070", "addr": "172.21.15.70", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:13:59.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:59 smithi070 ceph-mon[12201]: Deploying cephadm binary to smithi070 2024-08-31T02:13:59.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:13:59 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:13:59.628 INFO:teuthology.orchestra.run.smithi070.stderr:Deploying unmanaged mon service... 2024-08-31T02:14:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:00 smithi070 ceph-mon[12201]: Added host smithi070 2024-08-31T02:14:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:00 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:00.254 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: Scheduled mon update... 2024-08-31T02:14:00.954 INFO:teuthology.orchestra.run.smithi070.stderr:Deploying unmanaged mgr service... 2024-08-31T02:14:01.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:01 smithi070 ceph-mon[12201]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:01.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:01 smithi070 ceph-mon[12201]: Saving service mon spec with placement count:5 2024-08-31T02:14:01.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:01 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:01.594 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-08-31T02:14:02.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:02 smithi070 ceph-mon[12201]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:02.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:02 smithi070 ceph-mon[12201]: Saving service mgr spec with placement count:2 2024-08-31T02:14:02.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:02 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:04.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:04 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:04.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:04 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:04.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:04 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3545551850' entity='client.admin' 2024-08-31T02:14:05.967 INFO:teuthology.orchestra.run.smithi070.stderr:Enabling the dashboard module... 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='mgr.14120 172.21.15.70:0/1857619794' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]': finished 2024-08-31T02:14:06.169 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:05 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4040325197' entity='client.admin' 2024-08-31T02:14:07.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:06 smithi070 ceph-mon[12201]: Deploying daemon agent.smithi070 on smithi070 2024-08-31T02:14:07.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:06 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2839152111' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-31T02:14:07.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setuser ceph since I am not root 2024-08-31T02:14:07.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setgroup ceph since I am not root 2024-08-31T02:14:07.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.134+0000 7ff4e0f27000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:14:07.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.212+0000 7ff4e0f27000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:14:07.597 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.387+0000 7ff4e0f27000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:14:07.597 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.475+0000 7ff4e0f27000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:14:07.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.608+0000 7ff4e0f27000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:14:07.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:07.830+0000 7ff4e0f27000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:14:08.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2839152111' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-31T02:14:08.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:07 smithi070 ceph-mon[12201]: mgrmap e9: y(active, since 15s) 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "epoch": 9, 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "available": true, 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "active_name": "y", 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "num_standby": 0 2024-08-31T02:14:08.243 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:14:08.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:08.165+0000 7ff4e0f27000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:14:08.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:08.256+0000 7ff4e0f27000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:14:08.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:08.427+0000 7ff4e0f27000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:14:09.028 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:08 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2684075422' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-31T02:14:09.031 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for the mgr to restart... 2024-08-31T02:14:09.031 INFO:teuthology.orchestra.run.smithi070.stderr:Waiting for mgr epoch 9... 2024-08-31T02:14:09.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:09.496+0000 7ff4e0f27000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:14:09.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:09.584+0000 7ff4e0f27000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:14:10.778 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:10.471+0000 7ff4e0f27000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:14:11.096 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:10.776+0000 7ff4e0f27000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:14:11.096 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:10.908+0000 7ff4e0f27000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:14:11.096 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:10.996+0000 7ff4e0f27000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:14:11.096 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.094+0000 7ff4e0f27000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:14:11.424 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.182+0000 7ff4e0f27000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:14:11.424 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.329+0000 7ff4e0f27000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:14:11.424 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.422+0000 7ff4e0f27000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:14:11.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.518+0000 7ff4e0f27000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:14:12.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.802+0000 7ff4e0f27000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:14:12.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.891+0000 7ff4e0f27000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:14:12.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:11 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:11.980+0000 7ff4e0f27000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:14:13.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:14:13.043+0000 7ff4e0f27000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:14:14.026 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:14:14] ENGINE Bus STARTING 2024-08-31T02:14:14.106 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: { 2024-08-31T02:14:14.106 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-08-31T02:14:14.107 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: "initialized": true 2024-08-31T02:14:14.107 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: } 2024-08-31T02:14:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:14:14] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:14:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:14:14] ENGINE Bus STARTED 2024-08-31T02:14:14.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: Active manager daemon y restarted 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: Activating manager daemon y 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: osdmap e3: 0 total, 0 up, 0 in 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: mgrmap e10: y(active, starting, since 0.0523423s) 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:14.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:14:14.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:14:14.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:14 smithi070 ceph-mon[12201]: [31/Aug/2024:02:14:14] ENGINE Bus STARTING 2024-08-31T02:14:14.809 INFO:teuthology.orchestra.run.smithi070.stderr:mgr epoch 9 is available 2024-08-31T02:14:14.809 INFO:teuthology.orchestra.run.smithi070.stderr:Generating a dashboard self-signed certificate... 2024-08-31T02:14:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-31T02:14:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: mgrmap e11: y(active, since 1.0551s) 2024-08-31T02:14:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-31T02:14:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: [31/Aug/2024:02:14:14] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:14:15.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: [31/Aug/2024:02:14:14] ENGINE Bus STARTED 2024-08-31T02:14:15.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:15 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:15.491 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: Self-signed certificate created 2024-08-31T02:14:16.115 INFO:teuthology.orchestra.run.smithi070.stderr:Creating initial admin user... 2024-08-31T02:14:16.516 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:16.516 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:14:16.516 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: Deploying daemon agent.smithi070 on smithi070 2024-08-31T02:14:16.517 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:16 smithi070 ceph-mon[12201]: mgrmap e12: y(active, since 2s) 2024-08-31T02:14:16.938 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$BAMubpRUOyT3igH66BRuRuw7rj7wWvhZ.vnc2/Yww9GxXT0hnwTHy", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1725070456, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-31T02:14:17.539 INFO:teuthology.orchestra.run.smithi070.stderr:Fetching dashboard port number... 2024-08-31T02:14:18.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:17 smithi070 ceph-mon[12201]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:18.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:18.178 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: 8443 2024-08-31T02:14:18.814 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: disabled 2024-08-31T02:14:18.821 INFO:teuthology.orchestra.run.smithi070.stderr:systemctl: inactive 2024-08-31T02:14:18.822 INFO:teuthology.orchestra.run.smithi070.stderr:firewalld.service is not enabled 2024-08-31T02:14:18.822 INFO:teuthology.orchestra.run.smithi070.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr:Ceph Dashboard is now available at: 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr: URL: https://smithi070:8443/ 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr: User: admin 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr: Password: chvwawf9al 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:18.824 INFO:teuthology.orchestra.run.smithi070.stderr:Enabling autotune for osd_memory_target 2024-08-31T02:14:19.138 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:18 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1400517812' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-31T02:14:20.870 INFO:teuthology.orchestra.run.smithi070.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-08-31T02:14:21.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:20 smithi070 ceph-mon[12201]: mgrmap e13: y(active, since 6s) 2024-08-31T02:14:21.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:20 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2728964602' entity='client.admin' 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr:You can access the Ceph CLI with: 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr:Please consider enabling telemetry to help improve Ceph: 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.469 INFO:teuthology.orchestra.run.smithi070.stderr: ceph telemetry on 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr:For more information see: 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:14:21.470 INFO:teuthology.orchestra.run.smithi070.stderr:Bootstrap complete. 2024-08-31T02:14:21.647 INFO:tasks.cephadm:Fetching config... 2024-08-31T02:14:21.648 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:14:21.648 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-31T02:14:21.663 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-31T02:14:21.664 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:14:21.664 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-31T02:14:21.722 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-31T02:14:21.722 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:14:21.723 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-31T02:14:21.837 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-31T02:14:21.871 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:14:21.871 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-31T02:14:21.893 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-31T02:14:21.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwKb0OqbJ9lGiGbV+0lye719q5g+QoJ0uJxheeFeXBbjcOXAtTwT5unNZIQIiRmI8IOjMZzRawAaXGUIi+AM7xj8w7RZU6AUnXTIuCFAFmUD9JtseUgc5dX9NSEFZKh2Xwxu3PLTj32HamRavOhP1GrayRJ8ygZsdOeghXd8O3D28cJKSoZW9ncRB8lbOdji3n35qIAKQ5mnW8W6ukW0cCWetYN9i/W8nXsw/ZwLBQflW3EwZiDecSMCxuKtlZyNkGWx4araTWFWLNj4WFTZ40TodPg6Fly5Jyys0xrrXfQuCEJ4yVUXvKm8GWnys01EbrVyOYv0XVldIvGsLFO54XTrrToRmhfQeSsp9uFg7seIgsX4UMHd0hTkLVoVT+xsWblYWGzkLIybnXJ19jKJvuyQ7o5/6T5kTlvL/qBVGQwWIB9Xyzgj1X5Ohn9G3F/cISHw8Cm2YCbsDQTCKi0bRYA+aoGnd9Rpa03p4kpC9K85tN8qu3NhzerO6sadY4pys= ceph-862b0c56-673e-11ef-bcd4-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-31T02:14:22.098 INFO:teuthology.orchestra.run.smithi070.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwKb0OqbJ9lGiGbV+0lye719q5g+QoJ0uJxheeFeXBbjcOXAtTwT5unNZIQIiRmI8IOjMZzRawAaXGUIi+AM7xj8w7RZU6AUnXTIuCFAFmUD9JtseUgc5dX9NSEFZKh2Xwxu3PLTj32HamRavOhP1GrayRJ8ygZsdOeghXd8O3D28cJKSoZW9ncRB8lbOdji3n35qIAKQ5mnW8W6ukW0cCWetYN9i/W8nXsw/ZwLBQflW3EwZiDecSMCxuKtlZyNkGWx4araTWFWLNj4WFTZ40TodPg6Fly5Jyys0xrrXfQuCEJ4yVUXvKm8GWnys01EbrVyOYv0XVldIvGsLFO54XTrrToRmhfQeSsp9uFg7seIgsX4UMHd0hTkLVoVT+xsWblYWGzkLIybnXJ19jKJvuyQ7o5/6T5kTlvL/qBVGQwWIB9Xyzgj1X5Ohn9G3F/cISHw8Cm2YCbsDQTCKi0bRYA+aoGnd9Rpa03p4kpC9K85tN8qu3NhzerO6sadY4pys= ceph-862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:14:22.111 DEBUG:teuthology.orchestra.run.smithi102:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwKb0OqbJ9lGiGbV+0lye719q5g+QoJ0uJxheeFeXBbjcOXAtTwT5unNZIQIiRmI8IOjMZzRawAaXGUIi+AM7xj8w7RZU6AUnXTIuCFAFmUD9JtseUgc5dX9NSEFZKh2Xwxu3PLTj32HamRavOhP1GrayRJ8ygZsdOeghXd8O3D28cJKSoZW9ncRB8lbOdji3n35qIAKQ5mnW8W6ukW0cCWetYN9i/W8nXsw/ZwLBQflW3EwZiDecSMCxuKtlZyNkGWx4araTWFWLNj4WFTZ40TodPg6Fly5Jyys0xrrXfQuCEJ4yVUXvKm8GWnys01EbrVyOYv0XVldIvGsLFO54XTrrToRmhfQeSsp9uFg7seIgsX4UMHd0hTkLVoVT+xsWblYWGzkLIybnXJ19jKJvuyQ7o5/6T5kTlvL/qBVGQwWIB9Xyzgj1X5Ohn9G3F/cISHw8Cm2YCbsDQTCKi0bRYA+aoGnd9Rpa03p4kpC9K85tN8qu3NhzerO6sadY4pys= ceph-862b0c56-673e-11ef-bcd4-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-31T02:14:22.153 INFO:teuthology.orchestra.run.smithi102.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwKb0OqbJ9lGiGbV+0lye719q5g+QoJ0uJxheeFeXBbjcOXAtTwT5unNZIQIiRmI8IOjMZzRawAaXGUIi+AM7xj8w7RZU6AUnXTIuCFAFmUD9JtseUgc5dX9NSEFZKh2Xwxu3PLTj32HamRavOhP1GrayRJ8ygZsdOeghXd8O3D28cJKSoZW9ncRB8lbOdji3n35qIAKQ5mnW8W6ukW0cCWetYN9i/W8nXsw/ZwLBQflW3EwZiDecSMCxuKtlZyNkGWx4araTWFWLNj4WFTZ40TodPg6Fly5Jyys0xrrXfQuCEJ4yVUXvKm8GWnys01EbrVyOYv0XVldIvGsLFO54XTrrToRmhfQeSsp9uFg7seIgsX4UMHd0hTkLVoVT+xsWblYWGzkLIybnXJ19jKJvuyQ7o5/6T5kTlvL/qBVGQwWIB9Xyzgj1X5Ohn9G3F/cISHw8Cm2YCbsDQTCKi0bRYA+aoGnd9Rpa03p4kpC9K85tN8qu3NhzerO6sadY4pys= ceph-862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:14:22.165 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-31T02:14:22.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:14:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:23.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:23.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:25.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:25 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2174882638' entity='client.admin' 2024-08-31T02:14:25.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:25.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:26.558 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-31T02:14:26.558 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-31T02:14:26.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:14:26] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:27.559 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:14:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:29.576 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi102 2024-08-31T02:14:29.576 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:14:29.576 DEBUG:teuthology.orchestra.run.smithi102:> dd of=/etc/ceph/ceph.conf 2024-08-31T02:14:29.595 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:14:29.595 DEBUG:teuthology.orchestra.run.smithi102:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:14:29.653 INFO:tasks.cephadm:Adding host smithi102 to orchestrator... 2024-08-31T02:14:29.653 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch host add smithi102 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:30.190 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:29 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:31.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:30 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:14:32.374 INFO:teuthology.orchestra.run.smithi070.stdout:Added host 'smithi102' with addr '172.21.15.102' 2024-08-31T02:14:32.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:32 smithi070 ceph-mon[12201]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi102", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:32.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:32 smithi070 ceph-mon[12201]: Deploying cephadm binary to smithi102 2024-08-31T02:14:32.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch host ls --format=json 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: Added host smithi102 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:33.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:33.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:14:33.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:33.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:14:33.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]': finished 2024-08-31T02:14:34.791 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:14:34.791 INFO:teuthology.orchestra.run.smithi070.stdout:[{"addr": "172.21.15.70", "hostname": "smithi070", "labels": [], "status": ""}, {"addr": "172.21.15.102", "hostname": "smithi102", "labels": [], "status": ""}] 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:14:34] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: Deploying daemon agent.smithi102 on smithi102 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:35.068 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:35.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:35.350 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-31T02:14:35.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd crush tunables default 2024-08-31T02:14:35.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:14:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:36.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:35 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:14:36.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:35 smithi070 ceph-mon[12201]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:14:36.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:36.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:37.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:36 smithi070 ceph-mon[12201]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:37.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:14:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:38.000 INFO:teuthology.orchestra.run.smithi070.stderr:adjusted tunables profile to default 2024-08-31T02:14:38.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:38.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:38.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:37 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3552082968' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-31T02:14:38.615 INFO:tasks.cephadm:Adding mon.a on smithi070 2024-08-31T02:14:38.615 INFO:tasks.cephadm:Adding mon.c on smithi070 2024-08-31T02:14:38.615 INFO:tasks.cephadm:Adding mon.b on smithi102 2024-08-31T02:14:38.616 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply mon '3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi102:172.21.15.102=b' 2024-08-31T02:14:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:38 smithi070 ceph-mon[12201]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:38 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3552082968' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-31T02:14:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:38 smithi070 ceph-mon[12201]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:14:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:38 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:40.368 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled mon update... 2024-08-31T02:14:40.995 DEBUG:teuthology.orchestra.run.smithi070:mon.c> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.c.service 2024-08-31T02:14:40.997 DEBUG:teuthology.orchestra.run.smithi102:mon.b> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service 2024-08-31T02:14:41.000 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-31T02:14:41.000 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:14:41.125 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:40 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:42.046 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:42 smithi070 ceph-mon[12201]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi102:172.21.15.102=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:14:42.046 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:42 smithi070 ceph-mon[12201]: Saving service mon spec with placement smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi102:172.21.15.102=b;count:3 2024-08-31T02:14:42.046 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:42 smithi070 ceph-mon[12201]: Deploying daemon mon.c on smithi070 2024-08-31T02:14:42.857 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:14:42.857 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":1,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","modified":"2024-08-31T02:13:21.834661Z","created":"2024-08-31T02:13:21.834661Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-31T02:14:42.859 INFO:teuthology.orchestra.run.smithi102.stderr:dumped monmap epoch 1 2024-08-31T02:14:43.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:43 smithi070 ceph-mon[12201]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:43.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/2451493798' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:14:43.570 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:43 smithi070 ceph-mon[17138]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-08-31T02:14:44.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:43 smithi070 bash[17082]: 8d9f3f8a0f65aa74999b20b66a8e45e738a1c1cfa647967bb386f1ea1685c712 2024-08-31T02:14:44.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:43 smithi070 systemd[1]: Started Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:14:44.432 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-31T02:14:44.433 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:14:48] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: mon.a calling monitor election 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: mon.c calling monitor election 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: monmap e2: 2 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:14:48.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: mgrmap e13: y(active, since 35s) 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: overall HEALTH_OK 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:48.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: mon.c calling monitor election 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: monmap e2: 2 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: mgrmap e13: y(active, since 35s) 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: overall HEALTH_OK 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:48.954 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:48.955 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:14:48.955 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:48.955 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:49.165 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:14:49.165 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":2,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","modified":"2024-08-31T02:14:43.536532Z","created":"2024-08-31T02:13:21.834661Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3301","nonce":0},{"type":"v1","addr":"172.21.15.70:6790","nonce":0}]},"addr":"172.21.15.70:6790/0","public_addr":"172.21.15.70:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-31T02:14:49.167 INFO:teuthology.orchestra.run.smithi102.stderr:dumped monmap epoch 2 2024-08-31T02:14:49.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[12201]: Deploying daemon mon.b on smithi102 2024-08-31T02:14:49.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/1651654459' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:14:49.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:49.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[17138]: Deploying daemon mon.b on smithi102 2024-08-31T02:14:49.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/1651654459' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:14:49.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:49 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:50.961 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-31T02:14:50.961 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-31T02:14:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:50 smithi070 ceph-mon[12201]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:51.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:50 smithi070 ceph-mon[17138]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:53.319 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:14:53.320 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":2,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","modified":"2024-08-31T02:14:43.536532Z","created":"2024-08-31T02:13:21.834661Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3301","nonce":0},{"type":"v1","addr":"172.21.15.70:6790","nonce":0}]},"addr":"172.21.15.70:6790/0","public_addr":"172.21.15.70:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-31T02:14:53.321 INFO:teuthology.orchestra.run.smithi102.stderr:dumped monmap epoch 2 2024-08-31T02:14:53.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:52 smithi070 ceph-mon[12201]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:52 smithi070 ceph-mon[17138]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:53 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/98837416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:14:54.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:53 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/98837416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:14:54.956 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-31T02:14:54.956 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph mon dump -f json 2024-08-31T02:14:59.364 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:14:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:14:59.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:14:59.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: mon.c calling monitor election 2024-08-31T02:14:59.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:14:59.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: mgrmap e13: y(active, since 46s) 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: mon.b (rank 2) addr [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] is down (out of quorum) 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: mon.c calling monitor election 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: mon.a calling monitor election 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: mgrmap e13: y(active, since 46s) 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: mon.b (rank 2) addr [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] is down (out of quorum) 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:14:59.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:14:59.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:14:59.369 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:14:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:14:59.818 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:14:59.818 INFO:teuthology.orchestra.run.smithi102.stdout:{"epoch":3,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","modified":"2024-08-31T02:14:54.092049Z","created":"2024-08-31T02:13:21.834661Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3301","nonce":0},{"type":"v1","addr":"172.21.15.70:6790","nonce":0}]},"addr":"172.21.15.70:6790/0","public_addr":"172.21.15.70:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:3300","nonce":0},{"type":"v1","addr":"172.21.15.102:6789","nonce":0}]},"addr":"172.21.15.102:6789/0","public_addr":"172.21.15.102:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-31T02:14:59.819 INFO:teuthology.orchestra.run.smithi102.stderr:dumped monmap epoch 3 2024-08-31T02:15:00.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:00.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: Reconfiguring mon.a (unknown last config time)... 2024-08-31T02:15:00.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: Reconfiguring daemon mon.a on smithi070 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/1970997343' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:15:00.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:00.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: Reconfiguring mon.a (unknown last config time)... 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: Reconfiguring daemon mon.a on smithi070 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/1970997343' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-31T02:15:00.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:15:00.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:15:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:00.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:01.030 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-31T02:15:01.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph config generate-minimal-conf 2024-08-31T02:15:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mon.b calling monitor election 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mon.b calling monitor election 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mon.c calling monitor election 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mon.a calling monitor election 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: mgrmap e13: y(active, since 48s) 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: Cluster is now healthy 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: overall HEALTH_OK 2024-08-31T02:15:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mon.b calling monitor election 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mon.b calling monitor election 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mon.c calling monitor election 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:15:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: osdmap e4: 0 total, 0 up, 0 in 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: mgrmap e13: y(active, since 48s) 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: overall HEALTH_OK 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:02.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:02.864 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:02.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:02.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: Reconfiguring mon.c (monmap changed)... 2024-08-31T02:15:02.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: Reconfiguring daemon mon.c on smithi070 2024-08-31T02:15:02.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:15:02.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: Reconfiguring mon.c (monmap changed)... 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: Reconfiguring daemon mon.c on smithi070 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:02 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.282 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[12201]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:04.282 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.282 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.282 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.282 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:15:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:04.283 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[17138]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:04.283 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.283 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.283 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:04 smithi102 ceph-mon[13158]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:04.653 INFO:teuthology.orchestra.run.smithi070.stdout:# minimal ceph.conf for 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:15:04.653 INFO:teuthology.orchestra.run.smithi070.stdout:[global] 2024-08-31T02:15:04.653 INFO:teuthology.orchestra.run.smithi070.stdout: fsid = 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:15:04.653 INFO:teuthology.orchestra.run.smithi070.stdout: mon_host = [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] 2024-08-31T02:15:05.394 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:05 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2492535939' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:05.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:05 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2492535939' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:05.403 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:05 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2492535939' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:05.948 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-31T02:15:05.949 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:15:05.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.conf 2024-08-31T02:15:05.978 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:15:05.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:15:06.046 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:15:06.046 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.conf 2024-08-31T02:15:06.072 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:15:06.072 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:15:06.137 INFO:tasks.cephadm:Adding mgr.y on smithi070 2024-08-31T02:15:06.137 INFO:tasks.cephadm:Adding mgr.x on smithi102 2024-08-31T02:15:06.137 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply mgr '2;smithi070=y;smithi102=x' 2024-08-31T02:15:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[12201]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:06 smithi070 ceph-mon[17138]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:15:06.720 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:06.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:06 smithi102 ceph-mon[13158]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:15:08.038 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.039 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:08.039 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:08.039 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:08.039 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:08.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:08.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:08.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:08.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:08.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:08.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:08.466 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled mgr update... 2024-08-31T02:15:09.025 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:09.025 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.025 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:09.025 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.025 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:09.026 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.026 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:09.026 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-31T02:15:09.026 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:09.026 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.048 DEBUG:teuthology.orchestra.run.smithi102:mgr.x> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service 2024-08-31T02:15:09.050 INFO:tasks.cephadm:Deploying OSDs... 2024-08-31T02:15:09.050 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:15:09.050 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/scratch_devs of=/dev/stdout 2024-08-31T02:15:09.070 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-31T02:15:09.070 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_1 2024-08-31T02:15:09.129 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-31T02:15:09.129 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:09.129 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-08-31T02:15:09.129 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:09.129 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:09.130 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-08-31 02:15:03.245322910 +0000 2024-08-31T02:15:09.130 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-08-31 02:12:29.831635301 +0000 2024-08-31T02:15:09.130 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-08-31 02:12:29.831635301 +0000 2024-08-31T02:15:09.130 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-08-31 02:12:29.831635301 +0000 2024-08-31T02:15:09.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-31T02:15:09.194 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:09.195 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:09.196 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:09.196 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-08-31T02:15:09.197 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-08-31T02:15:09.197 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000128538 s, 4.0 MB/s 2024-08-31T02:15:09.198 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-31T02:15:09.257 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_2 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 682 Links: 1 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-08-31 02:15:03.245322910 +0000 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-08-31 02:12:30.171621836 +0000 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-08-31 02:12:30.171621836 +0000 2024-08-31T02:15:09.317 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-08-31 02:12:30.171621836 +0000 2024-08-31T02:15:09.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-31T02:15:09.383 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-08-31T02:15:09.383 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-08-31T02:15:09.383 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000118546 s, 4.3 MB/s 2024-08-31T02:15:09.384 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-31T02:15:09.441 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_3 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-08-31 02:15:03.245322910 +0000 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-08-31 02:12:30.486609362 +0000 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-08-31 02:12:30.486609362 +0000 2024-08-31T02:15:09.498 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-08-31 02:12:30.486609362 +0000 2024-08-31T02:15:09.499 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-31T02:15:09.563 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-08-31T02:15:09.563 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-08-31T02:15:09.563 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 8.9546e-05 s, 5.7 MB/s 2024-08-31T02:15:09.564 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-31T02:15:09.621 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_4 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 5h/5d Inode: 720 Links: 1 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:09.676 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2024-08-31 02:15:03.245322910 +0000 2024-08-31T02:15:09.677 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2024-08-31 02:12:30.798597006 +0000 2024-08-31T02:15:09.677 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2024-08-31 02:12:30.798597006 +0000 2024-08-31T02:15:09.677 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: 2024-08-31 02:12:30.798597006 +0000 2024-08-31T02:15:09.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-31T02:15:09.739 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2024-08-31T02:15:09.740 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2024-08-31T02:15:09.740 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000111969 s, 4.6 MB/s 2024-08-31T02:15:09.741 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-31T02:15:09.797 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:15:09.798 DEBUG:teuthology.orchestra.run.smithi102:> dd if=/scratch_devs of=/dev/stdout 2024-08-31T02:15:09.814 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-31T02:15:09.814 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_1 2024-08-31T02:15:09.869 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-31T02:15:09.869 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:09.869 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 664 Links: 1 2024-08-31T02:15:09.869 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:09.870 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:09.870 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-08-31 02:15:01.943915681 +0000 2024-08-31T02:15:09.870 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-08-31 02:12:29.979934762 +0000 2024-08-31T02:15:09.870 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-08-31 02:12:29.979934762 +0000 2024-08-31T02:15:09.870 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-08-31 02:12:29.979934762 +0000 2024-08-31T02:15:09.870 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-31T02:15:09.917 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:09 smithi102 ceph-mon[13158]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi102=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:09 smithi102 ceph-mon[13158]: Saving service mgr spec with placement smithi070=y;smithi102=x;count:2 2024-08-31T02:15:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:09 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:15:09.935 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-08-31T02:15:09.935 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-08-31T02:15:09.936 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000187364 s, 2.7 MB/s 2024-08-31T02:15:09.937 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-31T02:15:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[12201]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi102=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[12201]: Saving service mgr spec with placement smithi070=y;smithi102=x;count:2 2024-08-31T02:15:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:15:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[17138]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi070=y;smithi102=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[17138]: Saving service mgr spec with placement smithi070=y;smithi102=x;count:2 2024-08-31T02:15:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:09 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:15:09.993 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_2 2024-08-31T02:15:10.048 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-31T02:15:10.048 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:10.048 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-08-31 02:15:01.943915681 +0000 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-08-31 02:12:30.302927923 +0000 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-08-31 02:12:30.302927923 +0000 2024-08-31T02:15:10.049 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-08-31 02:12:30.302927923 +0000 2024-08-31T02:15:10.049 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-31T02:15:10.111 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-08-31T02:15:10.111 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-08-31T02:15:10.111 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000183768 s, 2.8 MB/s 2024-08-31T02:15:10.113 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-31T02:15:10.169 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_3 2024-08-31T02:15:10.224 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-31T02:15:10.224 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:10.224 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-08-31 02:15:01.943915681 +0000 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-08-31 02:12:30.632920936 +0000 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-08-31 02:12:30.632920936 +0000 2024-08-31T02:15:10.225 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-08-31 02:12:30.632920936 +0000 2024-08-31T02:15:10.225 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-31T02:15:10.419 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-08-31T02:15:10.419 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-08-31T02:15:10.419 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000182056 s, 2.8 MB/s 2024-08-31T02:15:10.420 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-31T02:15:10.436 DEBUG:teuthology.orchestra.run.smithi102:> stat /dev/vg_nvme/lv_4 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout:Access: 2024-08-31 02:15:10.376735487 +0000 2024-08-31T02:15:10.495 INFO:teuthology.orchestra.run.smithi102.stdout:Modify: 2024-08-31 02:12:30.959914012 +0000 2024-08-31T02:15:10.496 INFO:teuthology.orchestra.run.smithi102.stdout:Change: 2024-08-31 02:12:30.959914012 +0000 2024-08-31T02:15:10.496 INFO:teuthology.orchestra.run.smithi102.stdout: Birth: 2024-08-31 02:12:30.959914012 +0000 2024-08-31T02:15:10.496 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-31T02:15:10.595 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records in 2024-08-31T02:15:10.595 INFO:teuthology.orchestra.run.smithi102.stderr:1+0 records out 2024-08-31T02:15:10.595 INFO:teuthology.orchestra.run.smithi102.stderr:512 bytes copied, 0.000210173 s, 2.4 MB/s 2024-08-31T02:15:10.597 DEBUG:teuthology.orchestra.run.smithi102:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-31T02:15:10.612 INFO:tasks.cephadm:Deploying osd.0 on smithi070 with /dev/vg_nvme/lv_4... 2024-08-31T02:15:10.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-31T02:15:10.816 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:10 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:15:11.110 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:10 smithi102 ceph-mon[13158]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:10 smithi102 podman[14663]: 2024-08-31 02:15:10.81622664 +0000 UTC m=+0.105879498 container create 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, distribution-scope=public, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0) 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:10 smithi102 podman[14663]: 2024-08-31 02:15:10.726593891 +0000 UTC m=+0.016246763 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 podman[14663]: 2024-08-31 02:15:11.010166147 +0000 UTC m=+0.299819008 container init 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., version=8, GIT_CLEAN=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 podman[14663]: 2024-08-31 02:15:11.013042652 +0000 UTC m=+0.302695512 container start 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, name=centos-stream) 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-mgr[14682]: -- 172.21.15.102:0/2163957991 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ada6312340 con 0x55ada62ea800 2024-08-31T02:15:11.111 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 bash[14663]: 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed 2024-08-31T02:15:11.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:10 smithi070 ceph-mon[12201]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:11.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:10 smithi070 ceph-mon[17138]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:11.406 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:15:11.406 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.155+0000 7f5c22ea0000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:15:11.406 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.233+0000 7f5c22ea0000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:15:11.406 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.404+0000 7f5c22ea0000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:15:11.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.491+0000 7f5c22ea0000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:15:11.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.618+0000 7f5c22ea0000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:15:12.094 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:11.821+0000 7f5c22ea0000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:15:12.396 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:12.396 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.396 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: Reconfiguring mgr.y (unknown last config time)... 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-mon[13158]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:12.154+0000 7f5c22ea0000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:15:12.397 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:12.246+0000 7f5c22ea0000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:15:12.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:12.406 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: Reconfiguring mgr.y (unknown last config time)... 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[12201]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:15:12.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: Reconfiguring mgr.y (unknown last config time)... 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:12.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:12 smithi070 ceph-mon[17138]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:15:12.661 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:12.413+0000 7f5c22ea0000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:15:13.599 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:13.490+0000 7f5c22ea0000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:15:13.599 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:13.598+0000 7f5c22ea0000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:15:13.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:15:13.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:15:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:15:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:15:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:14.204 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:15:14.223 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_4 2024-08-31T02:15:14.743 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:14 smithi070 ceph-mon[12201]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:14.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:14 smithi070 ceph-mon[17138]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:14.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:14 smithi102 ceph-mon[13158]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:14.842 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:14.536+0000 7f5c22ea0000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:15:15.130 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:14.841+0000 7f5c22ea0000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:15:15.130 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:14.972+0000 7f5c22ea0000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:15:15.130 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.067+0000 7f5c22ea0000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:15:15.400 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.168+0000 7f5c22ea0000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:15:15.400 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.253+0000 7f5c22ea0000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:15:15.400 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.399+0000 7f5c22ea0000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:15:15.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:15.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:15.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.490+0000 7f5c22ea0000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:15:15.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.583+0000 7f5c22ea0000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.859+0000 7f5c22ea0000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:15:16.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:15.944+0000 7f5c22ea0000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:15:16.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:16.029+0000 7f5c22ea0000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:15:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:16.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:17.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[12201]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:17.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:17.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:17.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[17138]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:17.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:15:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:15:17.077+0000 7f5c22ea0000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:15:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:17 smithi102 ceph-mon[13158]: from='client.24115 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:15:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:18 smithi102 ceph-mon[13158]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:15:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:15:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:15:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:15:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[12201]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1891113056' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:15:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:18 smithi070 ceph-mon[17138]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]: dispatch 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]': finished 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: osdmap e5: 1 total, 0 up, 1 in 2024-08-31T02:15:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-31T02:15:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:15:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]: dispatch 2024-08-31T02:15:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]': finished 2024-08-31T02:15:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: osdmap e5: 1 total, 0 up, 1 in 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: mgrmap e14: y(active, since 65s), standbys: x 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]: dispatch 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3178488419' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e"}]': finished 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: osdmap e5: 1 total, 0 up, 1 in 2024-08-31T02:15:19.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:20.315 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:20 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1479105654' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:20.315 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:20 smithi070 ceph-mon[12201]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:20.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:20 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1479105654' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:20.315 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:20 smithi070 ceph-mon[17138]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:20 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1479105654' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:20 smithi102 ceph-mon[13158]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:22.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:22 smithi070 ceph-mon[12201]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:22.253 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:22 smithi070 ceph-mon[17138]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:22 smithi102 ceph-mon[13158]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:23.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:15:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:24 smithi102 ceph-mon[13158]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:24 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:24 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[12201]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[17138]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:24.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:24 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:26.129 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:15:26.129 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:26.129 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:15:26.129 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:26.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:15:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:27.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:27 smithi070 ceph-mon[12201]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:27.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:27 smithi070 ceph-mon[12201]: Deploying daemon osd.0 on smithi070 2024-08-31T02:15:27.254 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:27 smithi070 ceph-mon[17138]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:27.254 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:27 smithi070 ceph-mon[17138]: Deploying daemon osd.0 on smithi070 2024-08-31T02:15:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:27 smithi102 ceph-mon[13158]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:27 smithi102 ceph-mon[13158]: Deploying daemon osd.0 on smithi070 2024-08-31T02:15:29.338 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:29 smithi070 ceph-mon[12201]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:29.338 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:29 smithi070 ceph-mon[17138]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:29.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:29 smithi102 ceph-mon[13158]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:31.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:31.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:33.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:33 smithi102 ceph-mon[13158]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:33.186 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:33 smithi070 ceph-mon[12201]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:33.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:33 smithi070 ceph-mon[17138]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:33.216 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 0 on host 'smithi070' 2024-08-31T02:15:33.864 DEBUG:teuthology.orchestra.run.smithi070:osd.0> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service 2024-08-31T02:15:33.866 INFO:tasks.cephadm:Deploying osd.1 on smithi070 with /dev/vg_nvme/lv_3... 2024-08-31T02:15:33.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-31T02:15:34.509 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:34] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi070 to 15040M 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi070 to 15040M 2024-08-31T02:15:34.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi070 to 15040M 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.656 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:15:34.656 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:34.656 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:35.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[17138]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:15:35.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[17138]: osdmap e6: 1 total, 0 up, 1 in 2024-08-31T02:15:35.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:35.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[17138]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:35.391 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[17138]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:35.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[12201]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:15:35.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[12201]: osdmap e6: 1 total, 0 up, 1 in 2024-08-31T02:15:35.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:35.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[12201]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:35.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-mon[12201]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:35.392 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[20387]: 2024-08-31T02:15:35.280+0000 7fd7321e8700 -1 osd.0 0 waiting for initial osdmap 2024-08-31T02:15:35.392 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:15:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[20387]: 2024-08-31T02:15:35.282+0000 7fd72cb7f700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:15:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:35 smithi102 ceph-mon[13158]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:15:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:35 smithi102 ceph-mon[13158]: osdmap e6: 1 total, 0 up, 1 in 2024-08-31T02:15:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:35 smithi102 ceph-mon[13158]: from='osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:35 smithi102 ceph-mon[13158]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:36.468 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: osdmap e7: 1 total, 0 up, 1 in 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:36.489 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:36.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: osdmap e7: 1 total, 0 up, 1 in 2024-08-31T02:15:36.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.490 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_3 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: osdmap e7: 1 total, 0 up, 1 in 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:37.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:15:37.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:15:37.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[17138]: osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502] boot 2024-08-31T02:15:37.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[17138]: osdmap e8: 1 total, 1 up, 1 in 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[12201]: osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502] boot 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[12201]: osdmap e8: 1 total, 1 up, 1 in 2024-08-31T02:15:37.544 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:37 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:15:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:37 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:15:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:37 smithi102 ceph-mon[13158]: osd.0 [v2:172.21.15.70:6802/1973876502,v1:172.21.15.70:6803/1973876502] boot 2024-08-31T02:15:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:37 smithi102 ceph-mon[13158]: osdmap e8: 1 total, 1 up, 1 in 2024-08-31T02:15:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:15:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[12201]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[12201]: osdmap e9: 1 total, 1 up, 1 in 2024-08-31T02:15:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[17138]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[17138]: osdmap e9: 1 total, 1 up, 1 in 2024-08-31T02:15:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:38 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:38 smithi102 ceph-mon[13158]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-31T02:15:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:38 smithi102 ceph-mon[13158]: osdmap e9: 1 total, 1 up, 1 in 2024-08-31T02:15:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:38 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:15:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:38 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:15:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:38 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:39 smithi070 ceph-mon[12201]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:39 smithi070 ceph-mon[17138]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:39 smithi102 ceph-mon[13158]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:15:40.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:15:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:41.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[12201]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:41.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:41.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:41.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[17138]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:41 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:41 smithi102 ceph-mon[13158]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:41 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:41 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:42 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi070 2024-08-31T02:15:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:42 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]: dispatch 2024-08-31T02:15:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:42 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]': finished 2024-08-31T02:15:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:42 smithi102 ceph-mon[13158]: osdmap e10: 2 total, 1 up, 2 in 2024-08-31T02:15:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:42.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi070 2024-08-31T02:15:42.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]: dispatch 2024-08-31T02:15:42.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]': finished 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[12201]: osdmap e10: 2 total, 1 up, 2 in 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi070 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]: dispatch 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/172651070' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9fd4ae7d-6a0c-413a-82d8-3b49ba96388f"}]': finished 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[17138]: osdmap e10: 2 total, 1 up, 2 in 2024-08-31T02:15:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:43 smithi102 ceph-mon[13158]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:43 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2473610080' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:43.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:43 smithi070 ceph-mon[12201]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:43.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2473610080' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:43.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:43 smithi070 ceph-mon[17138]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:43.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:43 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2473610080' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:15:45.194 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:45 smithi070 ceph-mon[12201]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:45.194 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:45 smithi070 ceph-mon[17138]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:45.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:45 smithi102 ceph-mon[13158]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:47.058 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:47 smithi070 ceph-mon[12201]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:47.059 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:47 smithi070 ceph-mon[17138]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:47.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:47 smithi102 ceph-mon[13158]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:49.123 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[12201]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:49.124 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:15:49.124 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:49.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[17138]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:49.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:15:49.124 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:49 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:49 smithi102 ceph-mon[13158]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:49 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:15:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:49 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:50.347 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:50 smithi070 ceph-mon[12201]: Deploying daemon osd.1 on smithi070 2024-08-31T02:15:50.347 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:50 smithi070 ceph-mon[17138]: Deploying daemon osd.1 on smithi070 2024-08-31T02:15:50.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:50 smithi102 ceph-mon[13158]: Deploying daemon osd.1 on smithi070 2024-08-31T02:15:51.165 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:51 smithi070 ceph-mon[12201]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:51.166 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:51 smithi070 ceph-mon[17138]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:51 smithi102 ceph-mon[13158]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:53.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:53 smithi102 ceph-mon[13158]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:53.246 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:53 smithi070 ceph-mon[12201]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:53.246 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:53 smithi070 ceph-mon[17138]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:54.413 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:15:54] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:56.151 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 1 on host 'smithi070' 2024-08-31T02:15:56.773 DEBUG:teuthology.orchestra.run.smithi070:osd.1> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.1.service 2024-08-31T02:15:56.776 INFO:tasks.cephadm:Deploying osd.2 on smithi070 with /dev/vg_nvme/lv_2... 2024-08-31T02:15:56.776 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-31T02:15:57.302 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[12201]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:57 smithi070 ceph-mon[17138]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:15:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:15:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:57 smithi102 ceph-mon[13158]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-31T02:15:58.275 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi070 to 7520M 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[12201]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[12201]: osdmap e11: 2 total, 1 up, 2 in 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[12201]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi070 to 7520M 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[17138]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[17138]: osdmap e11: 2 total, 1 up, 2 in 2024-08-31T02:15:58.276 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:58.277 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-mon[17138]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:58.277 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1[23250]: 2024-08-31T02:15:58.173+0000 7fbcaeb58700 -1 osd.1 0 waiting for initial osdmap 2024-08-31T02:15:58.277 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:15:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1[23250]: 2024-08-31T02:15:58.176+0000 7fbca9cf0700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:15:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:58 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi070 to 7520M 2024-08-31T02:15:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:58 smithi102 ceph-mon[13158]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-31T02:15:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:58 smithi102 ceph-mon[13158]: osdmap e11: 2 total, 1 up, 2 in 2024-08-31T02:15:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:58 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:58 smithi102 ceph-mon[13158]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:15:59.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[12201]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[12201]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[12201]: osdmap e12: 2 total, 1 up, 2 in 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[17138]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[17138]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[17138]: osdmap e12: 2 total, 1 up, 2 in 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:59.146 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:15:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:59.384 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:15:59.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_2 2024-08-31T02:15:59.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:59 smithi102 ceph-mon[13158]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:15:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:59 smithi102 ceph-mon[13158]: from='osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:15:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:59 smithi102 ceph-mon[13158]: osdmap e12: 2 total, 1 up, 2 in 2024-08-31T02:15:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:15:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:15:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:16:00.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715] boot 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: osdmap e13: 2 total, 2 up, 2 in 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715] boot 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: osdmap e13: 2 total, 2 up, 2 in 2024-08-31T02:16:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: osd.1 [v2:172.21.15.70:6810/2753506715,v1:172.21.15.70:6811/2753506715] boot 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: osdmap e13: 2 total, 2 up, 2 in 2024-08-31T02:16:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:00 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:16:00.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:01.222 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: osdmap e14: 2 total, 2 up, 2 in 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi070 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.223 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: osdmap e14: 2 total, 2 up, 2 in 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi070 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:01.224 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: osdmap e14: 2 total, 2 up, 2 in 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi070 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[17138]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[17138]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[12201]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[12201]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:02 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:02 smithi102 ceph-mon[13158]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:02 smithi102 ceph-mon[13158]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:02 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:02 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:02 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:03 smithi070 ceph-mon[12201]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:03.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:03 smithi070 ceph-mon[17138]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:03 smithi102 ceph-mon[13158]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:04.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:05 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:05.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:05.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:05 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:06 smithi070 ceph-mon[12201]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:06 smithi070 ceph-mon[12201]: Cluster is now healthy 2024-08-31T02:16:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:06 smithi070 ceph-mon[17138]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:06 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:16:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:06 smithi102 ceph-mon[13158]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:06 smithi102 ceph-mon[13158]: Cluster is now healthy 2024-08-31T02:16:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]: dispatch 2024-08-31T02:16:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]': finished 2024-08-31T02:16:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: osdmap e15: 3 total, 2 up, 3 in 2024-08-31T02:16:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1943379940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]: dispatch 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]': finished 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: osdmap e15: 3 total, 2 up, 3 in 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1943379940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]: dispatch 2024-08-31T02:16:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2764814407' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9813ad1-db3a-424f-ba7b-9a155054a7db"}]': finished 2024-08-31T02:16:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: osdmap e15: 3 total, 2 up, 3 in 2024-08-31T02:16:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:07.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1943379940' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:09.679 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:09 smithi070 ceph-mon[12201]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:09.679 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:09 smithi070 ceph-mon[17138]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:09 smithi102 ceph-mon[13158]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:12.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:12 smithi102 ceph-mon[13158]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:12.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:12 smithi070 ceph-mon[12201]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:12.439 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:12 smithi070 ceph-mon[17138]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:13.264 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:16:13.264 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:13 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:13.264 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:16:13.264 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:13 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:16:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:13 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:14 smithi102 ceph-mon[13158]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:14 smithi102 ceph-mon[13158]: Deploying daemon osd.2 on smithi070 2024-08-31T02:16:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:16:14.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[12201]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[12201]: Deploying daemon osd.2 on smithi070 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[17138]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[17138]: Deploying daemon osd.2 on smithi070 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:16:14.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:16:14.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:16:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:15.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.401 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[12201]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:15.401 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.401 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.401 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:15 smithi070 ceph-mon[17138]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:15 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:15 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:15 smithi102 ceph-mon[13158]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:17.732 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:17 smithi070 ceph-mon[12201]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:17.732 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:17 smithi070 ceph-mon[17138]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:17.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:17 smithi102 ceph-mon[13158]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:19.018 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:19.018 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:19.018 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:19.018 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:19.018 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:19.019 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:19.019 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:19.019 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:19.019 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:19.019 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:18 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:19.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:18 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:18 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:18 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:18 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:18 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:20.099 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:19 smithi070 ceph-mon[12201]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:20.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:19 smithi070 ceph-mon[17138]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:19 smithi102 ceph-mon[13158]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:20.388 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 2 on host 'smithi070' 2024-08-31T02:16:21.033 DEBUG:teuthology.orchestra.run.smithi070:osd.2> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.2.service 2024-08-31T02:16:21.036 INFO:tasks.cephadm:Deploying osd.3 on smithi070 with /dev/vg_nvme/lv_1... 2024-08-31T02:16:21.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-31T02:16:21.623 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi070 to 5013M 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:21.624 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[12201]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi070 to 5013M 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.625 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.626 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:21.626 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:21.626 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:21 smithi070 ceph-mon[17138]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi070 to 5013M 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:21 smithi102 ceph-mon[13158]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-31T02:16:22.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[12201]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-31T02:16:22.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[12201]: osdmap e16: 3 total, 2 up, 3 in 2024-08-31T02:16:22.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:22.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[12201]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[12201]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[17138]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[17138]: osdmap e16: 3 total, 2 up, 3 in 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[17138]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:22.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-mon[17138]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:22.600 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2[26174]: 2024-08-31T02:16:22.417+0000 7f8133dba700 -1 osd.2 0 waiting for initial osdmap 2024-08-31T02:16:22.601 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2[26174]: 2024-08-31T02:16:22.420+0000 7f812c74d700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:16:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:22 smithi102 ceph-mon[13158]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-31T02:16:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:22 smithi102 ceph-mon[13158]: osdmap e16: 3 total, 2 up, 3 in 2024-08-31T02:16:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:22 smithi102 ceph-mon[13158]: from='osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:22 smithi102 ceph-mon[13158]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:23.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[17138]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:23.636 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[17138]: osdmap e17: 3 total, 2 up, 3 in 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[17138]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[12201]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[12201]: osdmap e17: 3 total, 2 up, 3 in 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:23 smithi070 ceph-mon[12201]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:23.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:23 smithi102 ceph-mon[13158]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:23.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:23 smithi102 ceph-mon[13158]: osdmap e17: 3 total, 2 up, 3 in 2024-08-31T02:16:23.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:23 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:23 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:23.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:23 smithi102 ceph-mon[13158]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-08-31T02:16:24.379 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:16:24.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi070:vg_nvme/lv_1 2024-08-31T02:16:24.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:24.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:16:24.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:16:24.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[12201]: osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081] boot 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[12201]: osdmap e18: 3 total, 3 up, 3 in 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[17138]: osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081] boot 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[17138]: osdmap e18: 3 total, 3 up, 3 in 2024-08-31T02:16:24.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:24 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:24.731 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:24 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:16:24.732 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:24 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:16:24.732 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:24 smithi102 ceph-mon[13158]: osd.2 [v2:172.21.15.70:6818/1249652081,v1:172.21.15.70:6819/1249652081] boot 2024-08-31T02:16:24.732 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:24 smithi102 ceph-mon[13158]: osdmap e18: 3 total, 3 up, 3 in 2024-08-31T02:16:24.732 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:24 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:16:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: osdmap e19: 3 total, 3 up, 3 in 2024-08-31T02:16:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: osdmap e19: 3 total, 3 up, 3 in 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: osdmap e19: 3 total, 3 up, 3 in 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:25.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:25.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-31T02:16:26.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[12201]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:26.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-31T02:16:26.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[12201]: osdmap e20: 3 total, 3 up, 3 in 2024-08-31T02:16:26.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-31T02:16:26.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[17138]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:26.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-31T02:16:26.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[17138]: osdmap e20: 3 total, 3 up, 3 in 2024-08-31T02:16:26.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:26 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-31T02:16:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:26 smithi102 ceph-mon[13158]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:16:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:26 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-31T02:16:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:26 smithi102 ceph-mon[13158]: osdmap e20: 3 total, 3 up, 3 in 2024-08-31T02:16:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:26 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-31T02:16:27.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: osdmap e21: 3 total, 3 up, 3 in 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: osdmap e21: 3 total, 3 up, 3 in 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:27 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: osdmap e21: 3 total, 3 up, 3 in 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:27 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:27.994 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27612]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:27.994 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27612]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:27.994 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27612]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:28.263 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27618]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:28.263 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27618]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:28.263 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:16:27 smithi070 sudo[27618]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:28.520 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27623]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:28.520 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27623]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:28.520 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27623]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[12201]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[12201]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[12201]: Cluster is now healthy 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[12201]: osdmap e22: 3 total, 3 up, 3 in 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[17138]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:28.521 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[17138]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:28.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:16:28.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:28.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:28 smithi070 ceph-mon[17138]: osdmap e22: 3 total, 3 up, 3 in 2024-08-31T02:16:28.781 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27626]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:28.781 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27626]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:28.781 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27626]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:28.781 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27630]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:28.781 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27630]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:28.782 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:28 smithi070 sudo[27630]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:28 smithi102 ceph-mon[13158]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:28 smithi102 ceph-mon[13158]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:16:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:28 smithi102 ceph-mon[13158]: Cluster is now healthy 2024-08-31T02:16:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:28 smithi102 ceph-mon[13158]: osdmap e22: 3 total, 3 up, 3 in 2024-08-31T02:16:29.125 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27650]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:29.125 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27650]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:29.125 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27650]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:29.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27656]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:29.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27656]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:29.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27656]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27665]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27665]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27665]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:30.047 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27681]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:30.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27681]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:30.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:29 smithi070 sudo[27681]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.308 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.309 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.309 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15651]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-31T02:16:30.309 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15651]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:30.309 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15651]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 sudo[27705]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 sudo[27705]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 sudo[27705]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:30.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:30 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:30.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15654]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-31T02:16:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15654]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-08-31T02:16:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:30 smithi102 sudo[15654]: pam_unix(sudo:session): session closed for user root 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]: dispatch 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]': finished 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: osdmap e23: 4 total, 3 up, 4 in 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:31.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:31.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:31.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]': finished 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: osdmap e23: 4 total, 3 up, 4 in 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2997169732' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed665570-237d-499e-8f15-2a48fb51ff92"}]': finished 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: osdmap e23: 4 total, 3 up, 4 in 2024-08-31T02:16:31.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:31.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:32 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/6452836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:32 smithi102 ceph-mon[13158]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:32 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/6452836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:32 smithi070 ceph-mon[12201]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:32 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/6452836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:32 smithi070 ceph-mon[17138]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:34.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:34 smithi102 ceph-mon[13158]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:34 smithi070 ceph-mon[12201]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:34 smithi070 ceph-mon[17138]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:34.812 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:16:34] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:35.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:35.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:35.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:35.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[12201]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 170 KiB/s wr, 12 op/s 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[17138]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 170 KiB/s wr, 12 op/s 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:16:37.312 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:37.339 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:37 smithi102 ceph-mon[13158]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 170 KiB/s wr, 12 op/s 2024-08-31T02:16:37.339 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:16:37.339 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:38.266 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:38 smithi070 ceph-mon[12201]: Deploying daemon osd.3 on smithi070 2024-08-31T02:16:38.266 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:38 smithi070 ceph-mon[17138]: Deploying daemon osd.3 on smithi070 2024-08-31T02:16:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:38 smithi102 ceph-mon[13158]: Deploying daemon osd.3 on smithi070 2024-08-31T02:16:39.297 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:39 smithi070 ceph-mon[12201]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 118 KiB/s wr, 8 op/s 2024-08-31T02:16:39.298 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:39 smithi070 ceph-mon[17138]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 118 KiB/s wr, 8 op/s 2024-08-31T02:16:39.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:39 smithi102 ceph-mon[13158]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 118 KiB/s wr, 8 op/s 2024-08-31T02:16:41.145 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:41 smithi070 ceph-mon[12201]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 91 KiB/s wr, 6 op/s 2024-08-31T02:16:41.145 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:41 smithi070 ceph-mon[17138]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 91 KiB/s wr, 6 op/s 2024-08-31T02:16:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:41 smithi102 ceph-mon[13158]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 91 KiB/s wr, 6 op/s 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:43.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2024-08-31T02:16:43.063 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:43.063 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:43.063 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:43.063 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:43.063 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:42 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:42 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:44.911 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:44 smithi070 ceph-mon[12201]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:44.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:44 smithi070 ceph-mon[17138]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:44 smithi102 ceph-mon[13158]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:45.713 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 3 on host 'smithi070' 2024-08-31T02:16:45.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:45.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:45.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:45.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:45.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:45 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:46.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:46.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:16:46.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:46.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:45 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:46.318 DEBUG:teuthology.orchestra.run.smithi070:osd.3> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.3.service 2024-08-31T02:16:46.321 INFO:tasks.cephadm:Deploying osd.4 on smithi102 with /dev/vg_nvme/lv_4... 2024-08-31T02:16:46.321 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-31T02:16:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi070 to 3760M 2024-08-31T02:16:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-31T02:16:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: osdmap e24: 4 total, 3 up, 4 in 2024-08-31T02:16:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:46 smithi102 ceph-mon[13158]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi070 to 3760M 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: osdmap e24: 4 total, 3 up, 4 in 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:47.006 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[12201]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi070 to 3760M 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: osdmap e24: 4 total, 3 up, 4 in 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: from='osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:47.007 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-mon[17138]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:16:47.007 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:16:46.807+0000 7f63e2c3b700 -1 osd.3 0 waiting for initial osdmap 2024-08-31T02:16:47.007 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:16:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:16:46.810+0000 7f63dcdd1700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:16:47.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:47.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:16:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: osdmap e25: 4 total, 3 up, 4 in 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:47.988 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:47.989 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:47 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: osdmap e25: 4 total, 3 up, 4 in 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: osdmap e25: 4 total, 3 up, 4 in 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:16:48.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:47 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:48.871 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:16:48.889 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi102:vg_nvme/lv_4 2024-08-31T02:16:48.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:16:48.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:16:48.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:48.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi070 2024-08-31T02:16:48.913 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044] boot 2024-08-31T02:16:48.913 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: osdmap e26: 4 total, 4 up, 4 in 2024-08-31T02:16:48.913 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:48 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:16:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:16:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi070 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044] boot 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: osdmap e26: 4 total, 4 up, 4 in 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi070 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: osd.3 [v2:172.21.15.70:6826/498008044,v1:172.21.15.70:6827/498008044] boot 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: osdmap e26: 4 total, 4 up, 4 in 2024-08-31T02:16:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:48 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:16:50.139 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:49 smithi102 ceph-mon[13158]: osdmap e27: 4 total, 4 up, 4 in 2024-08-31T02:16:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:49 smithi070 ceph-mon[12201]: osdmap e27: 4 total, 4 up, 4 in 2024-08-31T02:16:50.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:49 smithi070 ceph-mon[17138]: osdmap e27: 4 total, 4 up, 4 in 2024-08-31T02:16:51.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:51 smithi102 ceph-mon[13158]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:51.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:51.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:51.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:51.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[17138]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:51.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:51.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:51.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[12201]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:16:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:16:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:16:52.217 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:52 smithi102 ceph-mon[13158]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:52.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:52 smithi070 ceph-mon[17138]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:52 smithi070 ceph-mon[12201]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:16:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/51658591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]': finished 2024-08-31T02:16:53.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: osdmap e28: 5 total, 4 up, 5 in 2024-08-31T02:16:53.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:53 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:16:53.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/51658591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]': finished 2024-08-31T02:16:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: osdmap e28: 5 total, 4 up, 5 in 2024-08-31T02:16:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/51658591' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]: dispatch 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1166ee87-165d-4f2e-aa24-02fbad1fce50"}]': finished 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: osdmap e28: 5 total, 4 up, 5 in 2024-08-31T02:16:53.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:53 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:16:54.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:54 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/1644693712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:54.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:54 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/1644693712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:54 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/1644693712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:16:54.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:16:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:16:54] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:16:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:55 smithi102 ceph-mon[13158]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:55 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:55 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:55.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[12201]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:55.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:55.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:55.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[17138]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:55.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:55.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:55 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:16:57.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:57 smithi102 ceph-mon[13158]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:57.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:57 smithi070 ceph-mon[12201]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:57.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:57 smithi070 ceph-mon[17138]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:16:59 smithi102 ceph-mon[13158]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:16:59 smithi070 ceph-mon[12201]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:16:59.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:16:59 smithi070 ceph-mon[17138]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:01.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:01 smithi102 ceph-mon[13158]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:01.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:17:01.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:01 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[12201]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:17:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:01.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[17138]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:01.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:17:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:01 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:02.148 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:02 smithi102 ceph-mon[13158]: Deploying daemon osd.4 on smithi102 2024-08-31T02:17:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:02 smithi070 ceph-mon[12201]: Deploying daemon osd.4 on smithi102 2024-08-31T02:17:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:02 smithi070 ceph-mon[17138]: Deploying daemon osd.4 on smithi102 2024-08-31T02:17:03.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:03 smithi102 ceph-mon[13158]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:03 smithi070 ceph-mon[12201]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:03.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:03 smithi070 ceph-mon[17138]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:03.848 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:17:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[12201]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi070 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[17138]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:04.801 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi070 2024-08-31T02:17:04.802 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:04.891 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:04 smithi102 ceph-mon[13158]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:04.891 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:04.891 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:04 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi070 2024-08-31T02:17:04.891 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:06 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:06 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:07.943 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 4 on host 'smithi102' 2024-08-31T02:17:08.573 DEBUG:teuthology.orchestra.run.smithi102:osd.4> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.4.service 2024-08-31T02:17:08.616 INFO:tasks.cephadm:Deploying osd.5 on smithi102 with /dev/vg_nvme/lv_3... 2024-08-31T02:17:08.616 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi102 to 16064M 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.195 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.196 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:09.196 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:08 smithi102 ceph-mon[13158]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi102 to 16064M 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[12201]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi102 to 16064M 2024-08-31T02:17:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:08 smithi070 ceph-mon[17138]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-31T02:17:10.206 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-mon[13158]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-31T02:17:10.207 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-mon[13158]: osdmap e29: 5 total, 4 up, 5 in 2024-08-31T02:17:10.207 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:10.207 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-mon[13158]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:10.207 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-mon[13158]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:10.207 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:17:09.964+0000 7f583d57d700 -1 osd.4 0 waiting for initial osdmap 2024-08-31T02:17:10.207 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:17:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:17:09.969+0000 7f5839f18700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:17:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[12201]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-31T02:17:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[12201]: osdmap e29: 5 total, 4 up, 5 in 2024-08-31T02:17:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[12201]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[12201]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[17138]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[17138]: osdmap e29: 5 total, 4 up, 5 in 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[17138]: from='osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:09 smithi070 ceph-mon[17138]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: osdmap e30: 5 total, 4 up, 5 in 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623] boot 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: osdmap e31: 5 total, 5 up, 5 in 2024-08-31T02:17:11.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:10 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.293 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:17:11.312 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi102:vg_nvme/lv_3 2024-08-31T02:17:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: osdmap e30: 5 total, 4 up, 5 in 2024-08-31T02:17:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623] boot 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: osdmap e31: 5 total, 5 up, 5 in 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: osdmap e30: 5 total, 4 up, 5 in 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: osd.4 [v2:172.21.15.102:6800/245576623,v1:172.21.15.102:6801/245576623] boot 2024-08-31T02:17:11.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: osdmap e31: 5 total, 5 up, 5 in 2024-08-31T02:17:11.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:10 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:17:12.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:12 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:17:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:12 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:17:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:12 smithi102 ceph-mon[13158]: osdmap e32: 5 total, 5 up, 5 in 2024-08-31T02:17:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:17:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:17:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[12201]: osdmap e32: 5 total, 5 up, 5 in 2024-08-31T02:17:12.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:17:12.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:17:12.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:12 smithi070 ceph-mon[17138]: osdmap e32: 5 total, 5 up, 5 in 2024-08-31T02:17:13.159 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:13 smithi102 ceph-mon[13158]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:13.159 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:13 smithi102 ceph-mon[13158]: osdmap e33: 5 total, 5 up, 5 in 2024-08-31T02:17:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:13 smithi070 ceph-mon[12201]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:13 smithi070 ceph-mon[12201]: osdmap e33: 5 total, 5 up, 5 in 2024-08-31T02:17:13.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:13 smithi070 ceph-mon[17138]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:13 smithi070 ceph-mon[17138]: osdmap e33: 5 total, 5 up, 5 in 2024-08-31T02:17:14.336 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: osdmap e34: 5 total, 5 up, 5 in 2024-08-31T02:17:14.336 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:14.336 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:14.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:14.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:17:14.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:17:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:17:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: osdmap e34: 5 total, 5 up, 5 in 2024-08-31T02:17:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: osdmap e34: 5 total, 5 up, 5 in 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:17:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:17:15.226 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:15 smithi102 ceph-mon[13158]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:15.226 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:15 smithi102 ceph-mon[13158]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:15.226 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:15 smithi102 ceph-mon[13158]: osdmap e35: 5 total, 5 up, 5 in 2024-08-31T02:17:15.226 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:15 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:15.226 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:15 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[12201]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[12201]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[12201]: osdmap e35: 5 total, 5 up, 5 in 2024-08-31T02:17:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[17138]: from='client.34229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[17138]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[17138]: osdmap e35: 5 total, 5 up, 5 in 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:15 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi102 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/2168919600' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]': finished 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: osdmap e36: 6 total, 5 up, 6 in 2024-08-31T02:17:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi102 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/2168919600' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]': finished 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: osdmap e36: 6 total, 5 up, 6 in 2024-08-31T02:17:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:16 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi102 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/2168919600' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]: dispatch 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85292250-e58c-476f-8691-a5c47c0218f0"}]': finished 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: osdmap e36: 6 total, 5 up, 6 in 2024-08-31T02:17:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:16 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:17 smithi102 ceph-mon[13158]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-08-31T02:17:17.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:17 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/1555570889' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:17.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:17 smithi070 ceph-mon[12201]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-08-31T02:17:17.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:17 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/1555570889' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:17.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:17 smithi070 ceph-mon[17138]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-08-31T02:17:17.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:17 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/1555570889' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:19.254 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:19 smithi102 ceph-mon[13158]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-08-31T02:17:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:19 smithi070 ceph-mon[12201]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-08-31T02:17:19.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:19 smithi070 ceph-mon[17138]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-08-31T02:17:21.208 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:21 smithi102 ceph-mon[13158]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-08-31T02:17:21.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:21 smithi070 ceph-mon[12201]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-08-31T02:17:21.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:21 smithi070 ceph-mon[17138]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-08-31T02:17:23.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[12201]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:23.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:17:23.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:23.263 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[17138]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:23.264 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:17:23.264 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:23.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:23 smithi102 ceph-mon[13158]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:23.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:23 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:17:23.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:23 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:24.054 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:17:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:24.305 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[12201]: Deploying daemon osd.5 on smithi102 2024-08-31T02:17:24.305 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:24.306 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:24.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[17138]: Deploying daemon osd.5 on smithi102 2024-08-31T02:17:24.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:24.306 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:24 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:24.306 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:24 smithi102 ceph-mon[13158]: Deploying daemon osd.5 on smithi102 2024-08-31T02:17:24.307 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:24 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:24.307 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:24 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:25.333 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:25 smithi102 ceph-mon[13158]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-08-31T02:17:25.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:25 smithi070 ceph-mon[12201]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-08-31T02:17:25.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:25 smithi070 ceph-mon[17138]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-08-31T02:17:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:26 smithi102 ceph-mon[13158]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:26 smithi070 ceph-mon[12201]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:26 smithi070 ceph-mon[17138]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:28.935 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:28.936 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:28.936 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:28.936 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:28.936 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:28.936 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:28 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:28 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:30.257 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 5 on host 'smithi102' 2024-08-31T02:17:30.876 DEBUG:teuthology.orchestra.run.smithi102:osd.5> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.5.service 2024-08-31T02:17:30.879 INFO:tasks.cephadm:Deploying osd.6 on smithi102 with /dev/vg_nvme/lv_2... 2024-08-31T02:17:30.879 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:31.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:31 smithi102 ceph-mon[13158]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:31.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:31.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:31.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[17138]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:31.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:31.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:31 smithi070 ceph-mon[12201]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi102 to 8032M 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: osdmap e37: 6 total, 5 up, 6 in 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:32.152 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-mon[13158]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:32.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:17:32.280+0000 7f23a4c0c700 -1 osd.5 0 waiting for initial osdmap 2024-08-31T02:17:32.419 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:17:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:17:32.285+0000 7f23a05a5700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi102 to 8032M 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: osdmap e37: 6 total, 5 up, 6 in 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[12201]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi102 to 8032M 2024-08-31T02:17:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-31T02:17:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: osdmap e37: 6 total, 5 up, 6 in 2024-08-31T02:17:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: from='osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:32 smithi070 ceph-mon[17138]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:33.166 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:33 smithi102 ceph-mon[13158]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:33.166 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:33 smithi102 ceph-mon[13158]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:33.166 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:33 smithi102 ceph-mon[13158]: osdmap e38: 6 total, 5 up, 6 in 2024-08-31T02:17:33.166 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:33 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.166 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:33 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.438 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:17:33.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[12201]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:33.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[12201]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:33.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[12201]: osdmap e38: 6 total, 5 up, 6 in 2024-08-31T02:17:33.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[17138]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:33.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[17138]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:33.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[17138]: osdmap e38: 6 total, 5 up, 6 in 2024-08-31T02:17:33.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:33 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:33.456 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi102:vg_nvme/lv_2 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860] boot 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: osdmap e39: 6 total, 6 up, 6 in 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:34.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:34 smithi102 ceph-mon[13158]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-31T02:17:34.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:17:34] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860] boot 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: osdmap e39: 6 total, 6 up, 6 in 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[12201]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:17:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-08-31T02:17:34.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: osd.5 [v2:172.21.15.102:6808/1178177860,v1:172.21.15.102:6809/1178177860] boot 2024-08-31T02:17:34.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: osdmap e39: 6 total, 6 up, 6 in 2024-08-31T02:17:34.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:17:34.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:34 smithi070 ceph-mon[17138]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-31T02:17:35.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: osdmap e40: 6 total, 6 up, 6 in 2024-08-31T02:17:35.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.337 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi102 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:35.338 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:35 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: osdmap e40: 6 total, 6 up, 6 in 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi102 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:35.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: osdmap e40: 6 total, 6 up, 6 in 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi102 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:35.602 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:35 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: osdmap e41: 6 total, 6 up, 6 in 2024-08-31T02:17:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: osdmap e41: 6 total, 6 up, 6 in 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:36.560 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: osdmap e41: 6 total, 6 up, 6 in 2024-08-31T02:17:36.560 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:36.561 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:17:36.561 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:36.561 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:36.561 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:37.348 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:17:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: osdmap e42: 6 total, 6 up, 6 in 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:37.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:37.648 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: osdmap e42: 6 total, 6 up, 6 in 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: osdmap e42: 6 total, 6 up, 6 in 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:38 smithi102 ceph-mon[13158]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:38 smithi102 ceph-mon[13158]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:17:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:38 smithi102 ceph-mon[13158]: osdmap e43: 6 total, 6 up, 6 in 2024-08-31T02:17:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[12201]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[12201]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:17:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[12201]: osdmap e43: 6 total, 6 up, 6 in 2024-08-31T02:17:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[17138]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[17138]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:17:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:38 smithi070 ceph-mon[17138]: osdmap e43: 6 total, 6 up, 6 in 2024-08-31T02:17:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/3496699812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]': finished 2024-08-31T02:17:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: osdmap e44: 7 total, 6 up, 7 in 2024-08-31T02:17:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:39 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/2412114947' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/3496699812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]': finished 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: osdmap e44: 7 total, 6 up, 7 in 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/2412114947' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/3496699812' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]: dispatch 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b99f9a6a-26be-4986-ab23-b08baea904f3"}]': finished 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: osdmap e44: 7 total, 6 up, 7 in 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:39.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:39 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/2412114947' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:17:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:40 smithi102 ceph-mon[13158]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2024-08-31T02:17:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:40 smithi102 ceph-mon[13158]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-31T02:17:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:40 smithi102 ceph-mon[13158]: Cluster is now healthy 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[12201]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[12201]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[12201]: Cluster is now healthy 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[17138]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[17138]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-31T02:17:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:40 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:17:42.591 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:42 smithi070 ceph-mon[12201]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-08-31T02:17:42.592 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:42 smithi070 ceph-mon[17138]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-08-31T02:17:42.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:42 smithi102 ceph-mon[13158]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-08-31T02:17:44.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:17:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:44 smithi102 ceph-mon[13158]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-31T02:17:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:44 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:44 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[12201]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[17138]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:45.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:44 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:46.299 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:46 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:17:46.299 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:46 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:46 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:17:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:46 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:46.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:46 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:17:46.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:46 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:47.281 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:47 smithi102 ceph-mon[13158]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:47.281 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:47 smithi102 ceph-mon[13158]: Deploying daemon osd.6 on smithi102 2024-08-31T02:17:47.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:47 smithi070 ceph-mon[12201]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:47.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:47 smithi070 ceph-mon[12201]: Deploying daemon osd.6 on smithi102 2024-08-31T02:17:47.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:47 smithi070 ceph-mon[17138]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-08-31T02:17:47.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:47 smithi070 ceph-mon[17138]: Deploying daemon osd.6 on smithi102 2024-08-31T02:17:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:49 smithi102 ceph-mon[13158]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-08-31T02:17:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:49 smithi070 ceph-mon[12201]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-08-31T02:17:49.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:49 smithi070 ceph-mon[17138]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-08-31T02:17:51.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:51.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:51.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:51.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:51.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:51.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:51 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:51 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:53.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:53 smithi102 ceph-mon[13158]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:53.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:53 smithi070 ceph-mon[12201]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:53.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:53 smithi070 ceph-mon[17138]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:53.615 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 6 on host 'smithi102' 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:54 smithi102 ceph-mon[13158]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:54.277 DEBUG:teuthology.orchestra.run.smithi102:osd.6> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.6.service 2024-08-31T02:17:54.279 INFO:tasks.cephadm:Deploying osd.7 on smithi102 with /dev/vg_nvme/lv_1... 2024-08-31T02:17:54.279 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-31T02:17:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[12201]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:54 smithi070 ceph-mon[17138]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-31T02:17:55.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:55.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi102 to 5354M 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: osdmap e45: 7 total, 6 up, 7 in 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[17138]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi102 to 5354M 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: osdmap e45: 7 total, 6 up, 7 in 2024-08-31T02:17:55.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:55.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:55 smithi070 ceph-mon[12201]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi102 to 5354M 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: osdmap e45: 7 total, 6 up, 7 in 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: from='osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-mon[13158]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:17:55.736 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:17:55.633+0000 7fcf3242a700 -1 osd.6 0 waiting for initial osdmap 2024-08-31T02:17:55.736 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:17:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:17:55.639+0000 7fcf2adbd700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:17:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:56 smithi102 ceph-mon[13158]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:56 smithi102 ceph-mon[13158]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:56 smithi102 ceph-mon[13158]: osdmap e46: 7 total, 6 up, 7 in 2024-08-31T02:17:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:56 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:56 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[12201]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[12201]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[12201]: osdmap e46: 7 total, 6 up, 7 in 2024-08-31T02:17:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[17138]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:56.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[17138]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:17:56.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[17138]: osdmap e46: 7 total, 6 up, 7 in 2024-08-31T02:17:56.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:56 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:56.972 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-08-31T02:17:56.991 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch daemon add osd smithi102:vg_nvme/lv_1 2024-08-31T02:17:57.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:17:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:57.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:17:57.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:17:57.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595] boot 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: osdmap e47: 7 total, 7 up, 7 in 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:57.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:57 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595] boot 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: osdmap e47: 7 total, 7 up, 7 in 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:17:57.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: osd.6 [v2:172.21.15.102:6816/1131340595,v1:172.21.15.102:6817/1131340595] boot 2024-08-31T02:17:57.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: osdmap e47: 7 total, 7 up, 7 in 2024-08-31T02:17:57.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:17:57.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:57.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:57.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:57.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:57.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:57 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:58.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:58 smithi102 ceph-mon[13158]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:58 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi102 2024-08-31T02:17:58.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:58 smithi102 ceph-mon[13158]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:58 smithi102 ceph-mon[13158]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:58.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:58 smithi102 ceph-mon[13158]: osdmap e48: 7 total, 7 up, 7 in 2024-08-31T02:17:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[12201]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi102 2024-08-31T02:17:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[12201]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[12201]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[12201]: osdmap e48: 7 total, 7 up, 7 in 2024-08-31T02:17:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[17138]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi102 2024-08-31T02:17:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[17138]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-08-31T02:17:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[17138]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-08-31T02:17:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:58 smithi070 ceph-mon[17138]: osdmap e48: 7 total, 7 up, 7 in 2024-08-31T02:17:59.901 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: osdmap e49: 7 total, 7 up, 7 in 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:59.902 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:17:59 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:59.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:17:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:17:59.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: osdmap e49: 7 total, 7 up, 7 in 2024-08-31T02:17:59.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: osdmap e49: 7 total, 7 up, 7 in 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-31T02:17:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:17:59 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:00.887 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.887 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.888 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:00.888 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:18:00.888 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: Cluster is now healthy 2024-08-31T02:18:00.888 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:00 smithi102 ceph-mon[13158]: osdmap e50: 7 total, 7 up, 7 in 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:18:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[17138]: osdmap e50: 7 total, 7 up, 7 in 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: from='client.34283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi102:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: Cluster is now healthy 2024-08-31T02:18:00.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:00 smithi070 ceph-mon[12201]: osdmap e50: 7 total, 7 up, 7 in 2024-08-31T02:18:01.784 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:01 smithi102 ceph-mon[13158]: osdmap e51: 7 total, 7 up, 7 in 2024-08-31T02:18:01.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:01 smithi070 ceph-mon[12201]: osdmap e51: 7 total, 7 up, 7 in 2024-08-31T02:18:01.971 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:01 smithi070 ceph-mon[17138]: osdmap e51: 7 total, 7 up, 7 in 2024-08-31T02:18:02.839 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:02 smithi070 ceph-mon[12201]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:02.840 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:02 smithi070 ceph-mon[17138]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:02.915 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:02 smithi102 ceph-mon[13158]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:04.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:18:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/3366264393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]': finished 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: osdmap e52: 8 total, 7 up, 8 in 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-08-31T02:18:05.184 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/3366264393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]': finished 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: osdmap e52: 8 total, 7 up, 8 in 2024-08-31T02:18:05.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:04 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-08-31T02:18:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/3366264393' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]: dispatch 2024-08-31T02:18:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7415d75e-8e83-4d20-8b60-5b4607ed23af"}]': finished 2024-08-31T02:18:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: osdmap e52: 8 total, 7 up, 8 in 2024-08-31T02:18:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:04 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:06 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/3924276849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:18:06.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:06 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/3924276849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:18:06.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:06 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/3924276849' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-31T02:18:07.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:07 smithi070 ceph-mon[12201]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:07.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:07 smithi070 ceph-mon[17138]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:07.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:07 smithi102 ceph-mon[13158]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:09 smithi102 ceph-mon[13158]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-31T02:18:09.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:09 smithi070 ceph-mon[12201]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-31T02:18:09.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:09 smithi070 ceph-mon[17138]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-08-31T02:18:11.295 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:11 smithi102 ceph-mon[13158]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-31T02:18:11.295 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:11 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:18:11.295 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:11 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[12201]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-31T02:18:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:18:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[17138]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-31T02:18:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:18:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:11 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:12.156 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:12 smithi102 ceph-mon[13158]: Deploying daemon osd.7 on smithi102 2024-08-31T02:18:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:12 smithi070 ceph-mon[12201]: Deploying daemon osd.7 on smithi102 2024-08-31T02:18:12.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:12 smithi070 ceph-mon[17138]: Deploying daemon osd.7 on smithi102 2024-08-31T02:18:13.396 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:13 smithi102 ceph-mon[13158]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:13 smithi070 ceph-mon[12201]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:13.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:13 smithi070 ceph-mon[17138]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:14.266 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:14.267 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:14 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:14 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:14.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:14 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:15.345 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:15 smithi102 ceph-mon[13158]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:15 smithi070 ceph-mon[12201]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:15.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:15 smithi070 ceph-mon[17138]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:17.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:17.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:17.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:17.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:17.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:17 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:17.358 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:17 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:18.693 INFO:teuthology.orchestra.run.smithi102.stdout:Created osd(s) 7 on host 'smithi102' 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.350 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:19 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.370 DEBUG:teuthology.orchestra.run.smithi102:osd.7> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.7.service 2024-08-31T02:18:19.373 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-31T02:18:19.373 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd stat -f json 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:19.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.405 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:19.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:18:19] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:20.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi102 to 4016M 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi102 to 4016M 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:20.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:20.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:20.304 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:20 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi102 to 4016M 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:20.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:20 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: osdmap e53: 8 total, 7 up, 8 in 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-mon[13158]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.169 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:18:21.019+0000 7f0b892d2700 -1 osd.7 0 waiting for initial osdmap 2024-08-31T02:18:21.169 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:18:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:18:21.027+0000 7f0b80c63700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:18:21.328 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: osdmap e53: 8 total, 7 up, 8 in 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[12201]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-31T02:18:21.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: osdmap e53: 8 total, 7 up, 8 in 2024-08-31T02:18:21.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:21.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: from='osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-mon[17138]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]: dispatch 2024-08-31T02:18:21.442 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:18:21.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:18:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:22.145 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1725070676,"num_in_osds":8,"osd_in_since":1725070684,"num_remapped_pgs":0} 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: osdmap e54: 8 total, 7 up, 8 in 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:22.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:22.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:22.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:22 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2321656624' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: osdmap e54: 8 total, 7 up, 8 in 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2321656624' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi102", "root=default"]}]': finished 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: osdmap e54: 8 total, 7 up, 8 in 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:22.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:22.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:22.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:22.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:22 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2321656624' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:23.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd stat -f json 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: purged_snaps scrub starts 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: purged_snaps scrub ok 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500] boot 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: osdmap e55: 8 total, 8 up, 8 in 2024-08-31T02:18:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:23 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: purged_snaps scrub starts 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: purged_snaps scrub ok 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500] boot 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: osdmap e55: 8 total, 8 up, 8 in 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: purged_snaps scrub starts 2024-08-31T02:18:23.313 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: purged_snaps scrub ok 2024-08-31T02:18:23.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-08-31T02:18:23.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: osd.7 [v2:172.21.15.102:6824/2171508500,v1:172.21.15.102:6825/2171508500] boot 2024-08-31T02:18:23.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: osdmap e55: 8 total, 8 up, 8 in 2024-08-31T02:18:23.314 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:23 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:24.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:24 smithi070 ceph-mon[12201]: osdmap e56: 8 total, 8 up, 8 in 2024-08-31T02:18:24.359 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:24 smithi070 ceph-mon[17138]: osdmap e56: 8 total, 8 up, 8 in 2024-08-31T02:18:24.360 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:18:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:24 smithi102 ceph-mon[13158]: osdmap e56: 8 total, 8 up, 8 in 2024-08-31T02:18:25.355 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:25 smithi102 ceph-mon[13158]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:25.355 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:25 smithi102 ceph-mon[13158]: osdmap e57: 8 total, 8 up, 8 in 2024-08-31T02:18:25.355 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:25.355 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:25 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:25.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[12201]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:25.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[12201]: osdmap e57: 8 total, 8 up, 8 in 2024-08-31T02:18:25.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:25.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:25.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[17138]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:25.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[17138]: osdmap e57: 8 total, 8 up, 8 in 2024-08-31T02:18:25.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:25.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:25 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:26.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:26 smithi070 ceph-mon[12201]: osdmap e58: 8 total, 8 up, 8 in 2024-08-31T02:18:26.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:26 smithi070 ceph-mon[17138]: osdmap e58: 8 total, 8 up, 8 in 2024-08-31T02:18:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:26 smithi102 ceph-mon[13158]: osdmap e58: 8 total, 8 up, 8 in 2024-08-31T02:18:26.586 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[12201]: pgmap v179: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[12201]: osdmap e59: 8 total, 8 up, 8 in 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/388579201' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[17138]: pgmap v179: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[17138]: osdmap e59: 8 total, 8 up, 8 in 2024-08-31T02:18:27.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:27 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/388579201' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:27.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:27 smithi102 ceph-mon[13158]: pgmap v179: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:27 smithi102 ceph-mon[13158]: osdmap e59: 8 total, 8 up, 8 in 2024-08-31T02:18:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:27 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/388579201' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-31T02:18:27.762 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1725070702,"num_in_osds":8,"osd_in_since":1725070684,"num_remapped_pgs":0} 2024-08-31T02:18:27.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-31T02:18:27.986 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:18:29.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:29 smithi070 ceph-mon[12201]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-31T02:18:29.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:29 smithi070 ceph-mon[17138]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-31T02:18:29.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:29 smithi102 ceph-mon[13158]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-08-31T02:18:29.799 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:18:29.800 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":59,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","created":"2024-08-31T02:13:24.693365+0000","modified":"2024-08-31T02:18:26.036470+0000","last_up_change":"2024-08-31T02:18:22.013538+0000","last_in_change":"2024-08-31T02:18:04.676570+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T02:16:25.405107+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6803","nonce":1973876502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6805","nonce":1973876502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6809","nonce":1973876502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6807","nonce":1973876502}]},"public_addr":"172.21.15.70:6803/1973876502","cluster_addr":"172.21.15.70:6805/1973876502","heartbeat_back_addr":"172.21.15.70:6809/1973876502","heartbeat_front_addr":"172.21.15.70:6807/1973876502","state":["exists","up"]},{"osd":1,"uuid":"9fd4ae7d-6a0c-413a-82d8-3b49ba96388f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6811","nonce":2753506715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6813","nonce":2753506715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6817","nonce":2753506715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6815","nonce":2753506715}]},"public_addr":"172.21.15.70:6811/2753506715","cluster_addr":"172.21.15.70:6813/2753506715","heartbeat_back_addr":"172.21.15.70:6817/2753506715","heartbeat_front_addr":"172.21.15.70:6815/2753506715","state":["exists","up"]},{"osd":2,"uuid":"f9813ad1-db3a-424f-ba7b-9a155054a7db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6819","nonce":1249652081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6821","nonce":1249652081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6825","nonce":1249652081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6823","nonce":1249652081}]},"public_addr":"172.21.15.70:6819/1249652081","cluster_addr":"172.21.15.70:6821/1249652081","heartbeat_back_addr":"172.21.15.70:6825/1249652081","heartbeat_front_addr":"172.21.15.70:6823/1249652081","state":["exists","up"]},{"osd":3,"uuid":"ed665570-237d-499e-8f15-2a48fb51ff92","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6827","nonce":498008044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6829","nonce":498008044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6833","nonce":498008044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6831","nonce":498008044}]},"public_addr":"172.21.15.70:6827/498008044","cluster_addr":"172.21.15.70:6829/498008044","heartbeat_back_addr":"172.21.15.70:6833/498008044","heartbeat_front_addr":"172.21.15.70:6831/498008044","state":["exists","up"]},{"osd":4,"uuid":"1166ee87-165d-4f2e-aa24-02fbad1fce50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6801","nonce":245576623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6803","nonce":245576623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6807","nonce":245576623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6805","nonce":245576623}]},"public_addr":"172.21.15.102:6801/245576623","cluster_addr":"172.21.15.102:6803/245576623","heartbeat_back_addr":"172.21.15.102:6807/245576623","heartbeat_front_addr":"172.21.15.102:6805/245576623","state":["exists","up"]},{"osd":5,"uuid":"85292250-e58c-476f-8691-a5c47c0218f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6809","nonce":1178177860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6811","nonce":1178177860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6815","nonce":1178177860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6813","nonce":1178177860}]},"public_addr":"172.21.15.102:6809/1178177860","cluster_addr":"172.21.15.102:6811/1178177860","heartbeat_back_addr":"172.21.15.102:6815/1178177860","heartbeat_front_addr":"172.21.15.102:6813/1178177860","state":["exists","up"]},{"osd":6,"uuid":"b99f9a6a-26be-4986-ab23-b08baea904f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6817","nonce":1131340595}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6819","nonce":1131340595}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6823","nonce":1131340595}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6821","nonce":1131340595}]},"public_addr":"172.21.15.102:6817/1131340595","cluster_addr":"172.21.15.102:6819/1131340595","heartbeat_back_addr":"172.21.15.102:6823/1131340595","heartbeat_front_addr":"172.21.15.102:6821/1131340595","state":["exists","up"]},{"osd":7,"uuid":"7415d75e-8e83-4d20-8b60-5b4607ed23af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6825","nonce":2171508500}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6827","nonce":2171508500}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6831","nonce":2171508500}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6829","nonce":2171508500}]},"public_addr":"172.21.15.102:6825/2171508500","cluster_addr":"172.21.15.102:6827/2171508500","heartbeat_back_addr":"172.21.15.102:6831/2171508500","heartbeat_front_addr":"172.21.15.102:6829/2171508500","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:34.645449+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:57.486143+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:21.775305+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:45.902393+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:09.219360+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:31.756835+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:54.829523+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:18:20.114321+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/3589050309":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/3237384352":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6801/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6800/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/1047345415":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/848058187":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6801/3030506206":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/589807710":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/3374058595":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6800/3030506206":"2024-09-01T02:13:51.062496+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T02:18:30.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:30 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1109439128' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:18:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:30 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1109439128' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:18:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:30 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1109439128' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:18:30.556 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-31T02:16:25.405107+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-08-31T02:18:30.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-31T02:18:30.758 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:18:31.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:31 smithi070 ceph-mon[12201]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:31.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:31 smithi070 ceph-mon[17138]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:31.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:31 smithi102 ceph-mon[13158]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-08-31T02:18:32.451 INFO:teuthology.orchestra.run.smithi070.stdout:pg_num: 1 2024-08-31T02:18:33.193 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:33 smithi070 ceph-mon[12201]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-08-31T02:18:33.194 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:33 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2760053592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-31T02:18:33.194 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:33 smithi070 ceph-mon[17138]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-08-31T02:18:33.194 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:33 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2760053592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-31T02:18:33.218 INFO:tasks.cephadm:Adding prometheus.a on smithi102 2024-08-31T02:18:33.219 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply prometheus '1;smithi102=a' 2024-08-31T02:18:33.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:33 smithi102 ceph-mon[13158]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-08-31T02:18:33.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:33 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2760053592' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-31T02:18:35.157 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled prometheus update... 2024-08-31T02:18:35.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:35 smithi102 ceph-mon[13158]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-08-31T02:18:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:35 smithi070 ceph-mon[12201]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-08-31T02:18:35.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:35 smithi070 ceph-mon[17138]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-08-31T02:18:35.785 DEBUG:teuthology.orchestra.run.smithi102:prometheus.a> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service 2024-08-31T02:18:35.787 INFO:tasks.cephadm:Adding node-exporter.a on smithi070 2024-08-31T02:18:35.788 INFO:tasks.cephadm:Adding node-exporter.b on smithi102 2024-08-31T02:18:35.788 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply node-exporter '2;smithi070=a;smithi102=b' 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ignoring --setuser ceph since I am not root 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ignoring --setgroup ceph since I am not root 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mgr[14682]: -- 172.21.15.102:0/502777495 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bb34b10340 con 0x55bb34ae9000 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:36.354+0000 7feaba3bf000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: Saving service prometheus spec with placement smithi102=a;count:1 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-mon[13158]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setuser ceph since I am not root 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setgroup ceph since I am not root 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mgr[12432]: -- 172.21.15.70:0/215955528 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ec45c40340 con 0x55ec45c19000 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:36.360+0000 7fadfce0e000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:36.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: Saving service prometheus spec with placement smithi102=a;count:1 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[17138]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='client.34325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: Saving service prometheus spec with placement smithi102=a;count:1 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:36.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:36.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:36.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' 2024-08-31T02:18:36.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:36.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-mon[12201]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-31T02:18:36.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:36.439+0000 7fadfce0e000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:18:36.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:36.613+0000 7fadfce0e000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:18:36.699 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:36.436+0000 7feaba3bf000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:18:36.699 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:36.610+0000 7feaba3bf000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:18:36.700 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:36.698+0000 7feaba3bf000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:18:37.038 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:36.700+0000 7fadfce0e000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:18:37.039 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:36 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:36.830+0000 7fadfce0e000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:18:37.039 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:37.037+0000 7fadfce0e000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:18:37.041 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:36 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:36.831+0000 7feaba3bf000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:18:37.041 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:37.040+0000 7feaba3bf000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:18:37.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-mon[13158]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:37.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-mon[13158]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-31T02:18:37.363 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-mon[12201]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:37.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-mon[12201]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-31T02:18:37.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-mon[17138]: from='mgr.14152 172.21.15.70:0/1740437646' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:37.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-mon[17138]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-31T02:18:37.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:37.362+0000 7fadfce0e000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:18:37.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:37.449+0000 7fadfce0e000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:18:37.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:37.613+0000 7fadfce0e000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:18:37.644 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:37.384+0000 7feaba3bf000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:18:37.644 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:37.472+0000 7feaba3bf000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:18:37.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:37.644+0000 7feaba3bf000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:18:38.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:38.645+0000 7fadfce0e000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:18:38.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:38.735+0000 7fadfce0e000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:18:38.967 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:38.710+0000 7feaba3bf000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:18:38.968 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:38.799+0000 7feaba3bf000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:18:39.933 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:39.622+0000 7fadfce0e000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:18:39.974 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:39.705+0000 7feaba3bf000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:18:40.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:39.931+0000 7fadfce0e000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:18:40.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.064+0000 7fadfce0e000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:18:40.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.152+0000 7fadfce0e000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:18:40.229 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.023+0000 7feaba3bf000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:18:40.229 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.159+0000 7feaba3bf000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:18:40.508 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.251+0000 7fadfce0e000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:18:40.508 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.338+0000 7fadfce0e000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:18:40.508 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.505+0000 7fadfce0e000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:18:40.596 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.248+0000 7feaba3bf000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:18:40.596 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.349+0000 7feaba3bf000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:18:40.597 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.441+0000 7feaba3bf000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:18:40.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.595+0000 7feaba3bf000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:18:40.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.692+0000 7feaba3bf000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:18:40.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:40.791+0000 7feaba3bf000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:18:40.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.604+0000 7fadfce0e000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:18:40.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.701+0000 7fadfce0e000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:18:41.261 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:41.083+0000 7feaba3bf000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:18:41.261 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:41.171+0000 7feaba3bf000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:18:41.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:40.983+0000 7fadfce0e000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:18:41.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:41.071+0000 7fadfce0e000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:18:41.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:41.161+0000 7fadfce0e000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:18:41.544 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:41.261+0000 7feaba3bf000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:18:42.484 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[12201]: Active manager daemon y restarted 2024-08-31T02:18:42.484 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[12201]: Activating manager daemon y 2024-08-31T02:18:42.484 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[12201]: osdmap e60: 8 total, 8 up, 8 in 2024-08-31T02:18:42.484 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:42.220+0000 7fadfce0e000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:18:42.485 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:42] ENGINE Bus STARTING 2024-08-31T02:18:42.485 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[17138]: Active manager daemon y restarted 2024-08-31T02:18:42.485 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[17138]: Activating manager daemon y 2024-08-31T02:18:42.485 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-mon[17138]: osdmap e60: 8 total, 8 up, 8 in 2024-08-31T02:18:42.570 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-mon[13158]: Active manager daemon y restarted 2024-08-31T02:18:42.570 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-mon[13158]: Activating manager daemon y 2024-08-31T02:18:42.570 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-mon[13158]: osdmap e60: 8 total, 8 up, 8 in 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:42.345+0000 7feaba3bf000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: [31/Aug/2024:02:18:42] ENGINE Bus STARTING 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: CherryPy Checker: 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: The Application mounted at '' has an empty config. 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: [31/Aug/2024:02:18:42] ENGINE Serving on http://:::9283 2024-08-31T02:18:42.571 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:18:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: [31/Aug/2024:02:18:42] ENGINE Bus STARTED 2024-08-31T02:18:42.736 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: CherryPy Checker: 2024-08-31T02:18:42.736 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: The Application mounted at '' has an empty config. 2024-08-31T02:18:42.736 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:18:42.736 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:42] ENGINE Serving on http://:::9283 2024-08-31T02:18:42.737 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:42] ENGINE Bus STARTED 2024-08-31T02:18:43.305 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled node-exporter update... 2024-08-31T02:18:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: mgrmap e17: y(active, starting, since 0.0564049s), standbys: x 2024-08-31T02:18:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:18:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:18:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:18:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Manager daemon y is now available 2024-08-31T02:18:43.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:18:43.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: mgrmap e17: y(active, starting, since 0.0564049s), standbys: x 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:18:43.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.537 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.538 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: mgrmap e17: y(active, starting, since 0.0564049s), standbys: x 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:18:43.539 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Manager daemon y is now available 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:43.540 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:18:43.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2794818572' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:43.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-31T02:18:43.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:43] ENGINE Bus STARTING 2024-08-31T02:18:43.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:43] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:18:43.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:18:43] ENGINE Bus STARTED 2024-08-31T02:18:43.964 DEBUG:teuthology.orchestra.run.smithi070:node-exporter.a> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service 2024-08-31T02:18:43.967 DEBUG:teuthology.orchestra.run.smithi102:node-exporter.b> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service 2024-08-31T02:18:43.969 INFO:tasks.cephadm:Adding alertmanager.a on smithi070 2024-08-31T02:18:43.969 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply alertmanager '1;smithi070=a' 2024-08-31T02:18:44.291 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:18:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: mgrmap e18: y(active, since 1.07053s), standbys: x 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi102=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: Saving service node-exporter spec with placement smithi070=a;smithi102=b;count:2 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:18:44.597 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: [31/Aug/2024:02:18:43] ENGINE Bus STARTING 2024-08-31T02:18:44.598 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: [31/Aug/2024:02:18:43] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:18:44.598 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: [31/Aug/2024:02:18:43] ENGINE Bus STARTED 2024-08-31T02:18:44.598 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.598 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.598 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:44 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: mgrmap e18: y(active, since 1.07053s), standbys: x 2024-08-31T02:18:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi102=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: Saving service node-exporter spec with placement smithi070=a;smithi102=b;count:2 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: [31/Aug/2024:02:18:43] ENGINE Bus STARTING 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: [31/Aug/2024:02:18:43] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: [31/Aug/2024:02:18:43] ENGINE Bus STARTED 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: mgrmap e18: y(active, since 1.07053s), standbys: x 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi070=a;smithi102=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: Saving service node-exporter spec with placement smithi070=a;smithi102=b;count:2 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: [31/Aug/2024:02:18:43] ENGINE Bus STARTING 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: [31/Aug/2024:02:18:43] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: [31/Aug/2024:02:18:43] ENGINE Bus STARTED 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:44 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:46.256 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:46 smithi102 ceph-mon[13158]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:46.257 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:46 smithi102 ceph-mon[13158]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-31T02:18:46.420 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:46 smithi070 ceph-mon[17138]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:46.420 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:46 smithi070 ceph-mon[17138]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-31T02:18:46.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:46 smithi070 ceph-mon[12201]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:46.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:46 smithi070 ceph-mon[12201]: mgrmap e19: y(active, since 2s), standbys: x 2024-08-31T02:18:46.454 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled alertmanager update... 2024-08-31T02:18:47.459 DEBUG:teuthology.orchestra.run.smithi070:alertmanager.a> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service 2024-08-31T02:18:47.461 INFO:tasks.cephadm:Adding grafana.a on smithi102 2024-08-31T02:18:47.461 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph orch apply grafana '1;smithi102=a' 2024-08-31T02:18:47.761 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:47 smithi102 ceph-mon[13158]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:47.762 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:47 smithi102 ceph-mon[13158]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:47.762 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:47 smithi102 ceph-mon[13158]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-08-31T02:18:47.762 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:47 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:47.883 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[12201]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[12201]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[12201]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[17138]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[17138]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[17138]: Saving service alertmanager spec with placement smithi070=a;count:1 2024-08-31T02:18:47.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:47 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:49.746 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:49 smithi102 ceph-mon[13158]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:49.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:49 smithi070 ceph-mon[12201]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:49.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:49 smithi070 ceph-mon[17138]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:50.384 INFO:teuthology.orchestra.run.smithi102.stdout:Scheduled grafana update... 2024-08-31T02:18:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:51 smithi102 ceph-mon[13158]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:51 smithi102 ceph-mon[13158]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:51 smithi102 ceph-mon[13158]: Saving service grafana spec with placement smithi102=a;count:1 2024-08-31T02:18:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[12201]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[12201]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[12201]: Saving service grafana spec with placement smithi102=a;count:1 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[17138]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:51.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[17138]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi102=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:18:51.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[17138]: Saving service grafana spec with placement smithi102=a;count:1 2024-08-31T02:18:51.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:51.874 DEBUG:teuthology.orchestra.run.smithi102:grafana.a> sudo journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service 2024-08-31T02:18:51.877 INFO:tasks.cephadm:Setting up client nodes... 2024-08-31T02:18:51.877 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-31T02:18:53.498 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:53 smithi070 ceph-mon[12201]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:53.498 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:53 smithi070 ceph-mon[17138]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:53 smithi102 ceph-mon[13158]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:53.988 INFO:teuthology.orchestra.run.smithi070.stdout:[client.0] 2024-08-31T02:18:53.988 INFO:teuthology.orchestra.run.smithi070.stdout: key = AQCNfdJmruq1OhAArIPmE0QX/QKkjJ1hQLwnkQ== 2024-08-31T02:18:54.612 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4194143665' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.613 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.613 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:54.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4194143665' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:54 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:54.642 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2024-08-31T02:18:54.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-31T02:18:54.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-31T02:18:54.666 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:54 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4194143665' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.666 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:54 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:54.666 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:54 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:54.680 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-31T02:18:56.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:56 smithi102 ceph-mon[13158]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:56 smithi070 ceph-mon[12201]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:56.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:56 smithi070 ceph-mon[17138]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:56.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:56 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:18:57.215 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 podman[29361]: 2024-08-31 02:18:57.097702753 +0000 UTC m=+0.122868767 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:18:57.458 INFO:teuthology.orchestra.run.smithi102.stdout:[client.1] 2024-08-31T02:18:57.458 INFO:teuthology.orchestra.run.smithi102.stdout: key = AQCRfdJmYY8LGxAAekmgGDx+ELj3IkjrmuywJQ== 2024-08-31T02:18:57.469 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 podman[29361]: 2024-08-31 02:18:57.315423534 +0000 UTC m=+0.340589542 container create 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 podman[29361]: 2024-08-31 02:18:57.542586428 +0000 UTC m=+0.567752437 container init 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 podman[29361]: 2024-08-31 02:18:57.545519941 +0000 UTC m=+0.570685950 container start 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.569Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.570Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.571Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.571Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:18:57.855 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.574Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.574Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.869µs 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.574Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.574Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.574Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.053µs wal_replay_duration=339.538µs total_replay_duration=373.642µs 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.575Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.575Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.575Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.591Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.736843ms db_storage=510ns remote_storage=1.151µs web_handler=726ns query_engine=620ns scrape=444.245µs scrape_sd=17.784µs notify=829ns notify_sd=1.351µs rules=15.056912ms 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:18:57.591Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 bash[29361]: 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 2024-08-31T02:18:57.856 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:18:57 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='client.? 172.21.15.102:0/1216059853' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.314 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-08-31T02:18:58.314 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-31T02:18:58.314 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-31T02:18:58.351 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-31T02:18:58.351 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-31T02:18:58.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph mgr dump --format=json 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='client.? 172.21.15.102:0/1216059853' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.375 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='client.? 172.21.15.102:0/1216059853' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:58.376 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:18:58.377 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:58.555 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:18:58.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:18:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:18:58] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:18:59 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.102:9095"}]: dispatch 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:18:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:18:59 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:00.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:00 smithi102 ceph-mon[13158]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:00.358 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:19:00.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:00 smithi070 ceph-mon[12201]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:00.359 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:00 smithi070 ceph-mon[17138]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:01.033 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":19,"active_gid":14439,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":3763771457},{"type":"v1","addr":"172.21.15.70:6801","nonce":3763771457}]},"active_addr":"172.21.15.70:6801/3763771457","active_change":"2024-08-31T02:18:42.222386+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14433,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.70:8443/","prometheus":"http://172.21.15.70:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":663153968}]},{"addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":2284893798}]},{"addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":3182173953}]},{"addrvec":[{"type":"v2","addr":"172.21.15.70:0","nonce":1880793210}]}]}} 2024-08-31T02:19:01.037 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-31T02:19:01.037 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-31T02:19:01.037 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-31T02:19:01.288 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:01.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:01 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2079525795' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:19:01.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:01.409 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:01 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2079525795' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:19:01.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:01 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2079525795' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:19:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:02 smithi102 ceph-mon[13158]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:02 smithi102 ceph-mon[13158]: Detected new or changed devices on smithi102 2024-08-31T02:19:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[12201]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[12201]: Detected new or changed devices on smithi102 2024-08-31T02:19:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[17138]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[17138]: Detected new or changed devices on smithi102 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:19:03.072 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":60,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","created":"2024-08-31T02:13:24.693365+0000","modified":"2024-08-31T02:18:42.222069+0000","last_up_change":"2024-08-31T02:18:22.013538+0000","last_in_change":"2024-08-31T02:18:04.676570+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T02:16:25.405107+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6803","nonce":1973876502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6805","nonce":1973876502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6809","nonce":1973876502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6807","nonce":1973876502}]},"public_addr":"172.21.15.70:6803/1973876502","cluster_addr":"172.21.15.70:6805/1973876502","heartbeat_back_addr":"172.21.15.70:6809/1973876502","heartbeat_front_addr":"172.21.15.70:6807/1973876502","state":["exists","up"]},{"osd":1,"uuid":"9fd4ae7d-6a0c-413a-82d8-3b49ba96388f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6811","nonce":2753506715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6813","nonce":2753506715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6817","nonce":2753506715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6815","nonce":2753506715}]},"public_addr":"172.21.15.70:6811/2753506715","cluster_addr":"172.21.15.70:6813/2753506715","heartbeat_back_addr":"172.21.15.70:6817/2753506715","heartbeat_front_addr":"172.21.15.70:6815/2753506715","state":["exists","up"]},{"osd":2,"uuid":"f9813ad1-db3a-424f-ba7b-9a155054a7db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6819","nonce":1249652081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6821","nonce":1249652081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6825","nonce":1249652081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6823","nonce":1249652081}]},"public_addr":"172.21.15.70:6819/1249652081","cluster_addr":"172.21.15.70:6821/1249652081","heartbeat_back_addr":"172.21.15.70:6825/1249652081","heartbeat_front_addr":"172.21.15.70:6823/1249652081","state":["exists","up"]},{"osd":3,"uuid":"ed665570-237d-499e-8f15-2a48fb51ff92","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6827","nonce":498008044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6829","nonce":498008044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6833","nonce":498008044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6831","nonce":498008044}]},"public_addr":"172.21.15.70:6827/498008044","cluster_addr":"172.21.15.70:6829/498008044","heartbeat_back_addr":"172.21.15.70:6833/498008044","heartbeat_front_addr":"172.21.15.70:6831/498008044","state":["exists","up"]},{"osd":4,"uuid":"1166ee87-165d-4f2e-aa24-02fbad1fce50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6801","nonce":245576623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6803","nonce":245576623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6807","nonce":245576623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6805","nonce":245576623}]},"public_addr":"172.21.15.102:6801/245576623","cluster_addr":"172.21.15.102:6803/245576623","heartbeat_back_addr":"172.21.15.102:6807/245576623","heartbeat_front_addr":"172.21.15.102:6805/245576623","state":["exists","up"]},{"osd":5,"uuid":"85292250-e58c-476f-8691-a5c47c0218f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6809","nonce":1178177860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6811","nonce":1178177860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6815","nonce":1178177860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6813","nonce":1178177860}]},"public_addr":"172.21.15.102:6809/1178177860","cluster_addr":"172.21.15.102:6811/1178177860","heartbeat_back_addr":"172.21.15.102:6815/1178177860","heartbeat_front_addr":"172.21.15.102:6813/1178177860","state":["exists","up"]},{"osd":6,"uuid":"b99f9a6a-26be-4986-ab23-b08baea904f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6817","nonce":1131340595}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6819","nonce":1131340595}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6823","nonce":1131340595}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6821","nonce":1131340595}]},"public_addr":"172.21.15.102:6817/1131340595","cluster_addr":"172.21.15.102:6819/1131340595","heartbeat_back_addr":"172.21.15.102:6823/1131340595","heartbeat_front_addr":"172.21.15.102:6821/1131340595","state":["exists","up"]},{"osd":7,"uuid":"7415d75e-8e83-4d20-8b60-5b4607ed23af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6825","nonce":2171508500}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6827","nonce":2171508500}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6831","nonce":2171508500}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6829","nonce":2171508500}]},"public_addr":"172.21.15.102:6825/2171508500","cluster_addr":"172.21.15.102:6827/2171508500","heartbeat_back_addr":"172.21.15.102:6831/2171508500","heartbeat_front_addr":"172.21.15.102:6829/2171508500","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:34.645449+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:57.486143+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:21.775305+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:45.902393+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:09.219360+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:31.756835+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:54.829523+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:18:20.114321+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2952173293":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3561013195":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3189512648":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3589050309":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6801/2881947222":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3237384352":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6801/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6800/2881947222":"2024-09-01T02:18:42.222037+0000","172.21.15.70:6800/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/1047345415":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/2643865740":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/848058187":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6801/3030506206":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/589807710":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/3374058595":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6800/3030506206":"2024-09-01T02:13:51.062496+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T02:19:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3756890146' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:03 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3756890146' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:03 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3756890146' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:03.894 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-31T02:19:03.895 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd dump --format=json 2024-08-31T02:19:04.122 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:04.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:04 smithi070 bash[33237]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-08-31T02:19:04.277 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:19:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:04.554 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:04 smithi070 bash[33237]: Getting image source signatures 2024-08-31T02:19:04.554 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:04 smithi070 bash[33237]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-08-31T02:19:04.554 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:04 smithi070 bash[33237]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-08-31T02:19:04.554 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:04 smithi070 bash[33237]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-08-31T02:19:05.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:05 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:05.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:05 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:05.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:05 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:05.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:05 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:05 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:05 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:06.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:06 smithi070 ceph-mon[12201]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:06.449 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:06 smithi070 bash[33237]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-08-31T02:19:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:06 smithi070 ceph-mon[17138]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:06 smithi102 ceph-mon[13158]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:06.856 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:06 smithi070 bash[33237]: Writing manifest to image destination 2024-08-31T02:19:07.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:06 smithi070 podman[33237]: 2024-08-31 02:19:06.898462463 +0000 UTC m=+2.985931649 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-08-31T02:19:07.561 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 podman[33237]: 2024-08-31 02:19:07.473115956 +0000 UTC m=+3.560585138 container create 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:07.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 podman[33237]: 2024-08-31 02:19:07.66718061 +0000 UTC m=+3.754649793 container init 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 podman[33237]: 2024-08-31 02:19:07.670639062 +0000 UTC m=+3.758108245 container start 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.674Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.674Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.674Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.674Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=arp 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=bcache 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=bonding 2024-08-31T02:19:07.950 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=btrfs 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=conntrack 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=cpu 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=diskstats 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=dmi 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=edac 2024-08-31T02:19:07.951 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=entropy 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=filefd 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=filesystem 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=hwmon 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=infiniband 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=ipvs 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=loadavg 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=mdadm 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=meminfo 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=netclass 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=netdev 2024-08-31T02:19:07.952 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=netstat 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=nfs 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=nfsd 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=nvme 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=os 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=pressure 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=rapl 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=schedstat 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=sockstat 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=softnet 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=stat 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=tapestats 2024-08-31T02:19:07.953 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=textfile 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=time 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=uname 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=vmstat 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=xfs 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:115 level=info collector=zfs 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[33465]: ts=2024-08-31T02:19:07.675Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 bash[33237]: 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 2024-08-31T02:19:07.954 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:19:07 smithi070 systemd[1]: Started Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:19:08.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:08 smithi102 ceph-mon[13158]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:08.349 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:08.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:08 smithi070 ceph-mon[12201]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:08.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:08.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:08 smithi070 ceph-mon[17138]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:08.527 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:08.599 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:19:08.599 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":60,"fsid":"862b0c56-673e-11ef-bcd4-c7b262605968","created":"2024-08-31T02:13:24.693365+0000","modified":"2024-08-31T02:18:42.222069+0000","last_up_change":"2024-08-31T02:18:22.013538+0000","last_in_change":"2024-08-31T02:18:04.676570+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T02:16:25.405107+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"d5a369b0-bcb1-4f17-b82b-3cd2622f0e9e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6803","nonce":1973876502}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6805","nonce":1973876502}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6809","nonce":1973876502}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1973876502},{"type":"v1","addr":"172.21.15.70:6807","nonce":1973876502}]},"public_addr":"172.21.15.70:6803/1973876502","cluster_addr":"172.21.15.70:6805/1973876502","heartbeat_back_addr":"172.21.15.70:6809/1973876502","heartbeat_front_addr":"172.21.15.70:6807/1973876502","state":["exists","up"]},{"osd":1,"uuid":"9fd4ae7d-6a0c-413a-82d8-3b49ba96388f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6811","nonce":2753506715}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6813","nonce":2753506715}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6817","nonce":2753506715}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":2753506715},{"type":"v1","addr":"172.21.15.70:6815","nonce":2753506715}]},"public_addr":"172.21.15.70:6811/2753506715","cluster_addr":"172.21.15.70:6813/2753506715","heartbeat_back_addr":"172.21.15.70:6817/2753506715","heartbeat_front_addr":"172.21.15.70:6815/2753506715","state":["exists","up"]},{"osd":2,"uuid":"f9813ad1-db3a-424f-ba7b-9a155054a7db","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6819","nonce":1249652081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6821","nonce":1249652081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6825","nonce":1249652081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":1249652081},{"type":"v1","addr":"172.21.15.70:6823","nonce":1249652081}]},"public_addr":"172.21.15.70:6819/1249652081","cluster_addr":"172.21.15.70:6821/1249652081","heartbeat_back_addr":"172.21.15.70:6825/1249652081","heartbeat_front_addr":"172.21.15.70:6823/1249652081","state":["exists","up"]},{"osd":3,"uuid":"ed665570-237d-499e-8f15-2a48fb51ff92","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6827","nonce":498008044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6829","nonce":498008044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6832","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6833","nonce":498008044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":498008044},{"type":"v1","addr":"172.21.15.70:6831","nonce":498008044}]},"public_addr":"172.21.15.70:6827/498008044","cluster_addr":"172.21.15.70:6829/498008044","heartbeat_back_addr":"172.21.15.70:6833/498008044","heartbeat_front_addr":"172.21.15.70:6831/498008044","state":["exists","up"]},{"osd":4,"uuid":"1166ee87-165d-4f2e-aa24-02fbad1fce50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6800","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6801","nonce":245576623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6802","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6803","nonce":245576623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6806","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6807","nonce":245576623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6804","nonce":245576623},{"type":"v1","addr":"172.21.15.102:6805","nonce":245576623}]},"public_addr":"172.21.15.102:6801/245576623","cluster_addr":"172.21.15.102:6803/245576623","heartbeat_back_addr":"172.21.15.102:6807/245576623","heartbeat_front_addr":"172.21.15.102:6805/245576623","state":["exists","up"]},{"osd":5,"uuid":"85292250-e58c-476f-8691-a5c47c0218f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6808","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6809","nonce":1178177860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6810","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6811","nonce":1178177860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6814","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6815","nonce":1178177860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6812","nonce":1178177860},{"type":"v1","addr":"172.21.15.102:6813","nonce":1178177860}]},"public_addr":"172.21.15.102:6809/1178177860","cluster_addr":"172.21.15.102:6811/1178177860","heartbeat_back_addr":"172.21.15.102:6815/1178177860","heartbeat_front_addr":"172.21.15.102:6813/1178177860","state":["exists","up"]},{"osd":6,"uuid":"b99f9a6a-26be-4986-ab23-b08baea904f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6816","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6817","nonce":1131340595}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6818","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6819","nonce":1131340595}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6822","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6823","nonce":1131340595}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6820","nonce":1131340595},{"type":"v1","addr":"172.21.15.102:6821","nonce":1131340595}]},"public_addr":"172.21.15.102:6817/1131340595","cluster_addr":"172.21.15.102:6819/1131340595","heartbeat_back_addr":"172.21.15.102:6823/1131340595","heartbeat_front_addr":"172.21.15.102:6821/1131340595","state":["exists","up"]},{"osd":7,"uuid":"7415d75e-8e83-4d20-8b60-5b4607ed23af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6824","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6825","nonce":2171508500}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6826","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6827","nonce":2171508500}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6830","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6831","nonce":2171508500}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.102:6828","nonce":2171508500},{"type":"v1","addr":"172.21.15.102:6829","nonce":2171508500}]},"public_addr":"172.21.15.102:6825/2171508500","cluster_addr":"172.21.15.102:6827/2171508500","heartbeat_back_addr":"172.21.15.102:6831/2171508500","heartbeat_front_addr":"172.21.15.102:6829/2171508500","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:34.645449+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:15:57.486143+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:21.775305+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:16:45.902393+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:09.219360+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:31.756835+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:17:54.829523+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T02:18:20.114321+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/2952173293":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3561013195":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3189512648":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3589050309":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6801/2881947222":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/3237384352":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6801/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:6800/2881947222":"2024-09-01T02:18:42.222037+0000","172.21.15.70:6800/3252563261":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/1047345415":"2024-09-01T02:14:13.045435+0000","172.21.15.70:0/2643865740":"2024-09-01T02:18:42.222037+0000","172.21.15.70:0/848058187":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6801/3030506206":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/589807710":"2024-09-01T02:13:51.062496+0000","172.21.15.70:0/3374058595":"2024-09-01T02:13:51.062496+0000","172.21.15.70:6800/3030506206":"2024-09-01T02:13:51.062496+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T02:19:08.630 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:08 smithi102 systemd[1]: Starting Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:19:08.918 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:08 smithi102 bash[30231]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-08-31T02:19:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:09 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:19:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1541668246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:09.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:09 smithi102 bash[30231]: Getting image source signatures 2024-08-31T02:19:09.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:09 smithi102 bash[30231]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-08-31T02:19:09.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:09 smithi102 bash[30231]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-08-31T02:19:09.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:09 smithi102 bash[30231]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-08-31T02:19:09.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-31T02:19:09.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-31T02:19:09.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-31T02:19:09.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-31T02:19:09.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-31T02:19:09.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-31T02:19:09.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-31T02:19:09.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-31T02:19:09.518 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:09 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:19:09.518 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1541668246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:09.519 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:09 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:19:09.519 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1541668246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-31T02:19:10.317 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.318 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.319 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.320 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.320 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.363 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.373 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:10.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:10 smithi070 ceph-mon[12201]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:10.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:10 smithi070 ceph-mon[17138]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:10.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:10 smithi102 ceph-mon[13158]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:10.518 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:10 smithi102 bash[30231]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-08-31T02:19:10.518 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:10 smithi102 bash[30231]: Writing manifest to image destination 2024-08-31T02:19:11.242 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:10 smithi102 podman[30231]: 2024-08-31 02:19:10.933146898 +0000 UTC m=+2.396017727 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-08-31T02:19:11.242 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 podman[30231]: 2024-08-31 02:19:11.048192545 +0000 UTC m=+2.511063373 container create 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:19:11.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 podman[30231]: 2024-08-31 02:19:11.242107511 +0000 UTC m=+2.704978340 container init 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 podman[30231]: 2024-08-31 02:19:11.245475572 +0000 UTC m=+2.708346402 container start 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.250Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.250Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=arp 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=bcache 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=bonding 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=btrfs 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=conntrack 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=cpu 2024-08-31T02:19:11.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=diskstats 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=dmi 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=edac 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=entropy 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=filefd 2024-08-31T02:19:11.670 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=filesystem 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=hwmon 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=infiniband 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=ipvs 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=loadavg 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=mdadm 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=meminfo 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=netclass 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=netdev 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=netstat 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=nfs 2024-08-31T02:19:11.671 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=nfsd 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=nvme 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=os 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=pressure 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=rapl 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=schedstat 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=sockstat 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=softnet 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=stat 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=tapestats 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=textfile 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-08-31T02:19:11.672 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=time 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=uname 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=vmstat 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=xfs 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.251Z caller=node_exporter.go:115 level=info collector=zfs 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.252Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[30307]: ts=2024-08-31T02:19:11.252Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 bash[30231]: 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd 2024-08-31T02:19:11.673 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:19:11 smithi102 systemd[1]: Started Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:19:12.402 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:12 smithi070 ceph-mon[12201]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:12.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:12 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:12.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:12 smithi070 ceph-mon[17138]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:12.403 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:12 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:12.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:12 smithi102 ceph-mon[13158]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:12.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:12 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:13.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:13 smithi102 ceph-mon[13158]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:19:13.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:13 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:13 smithi070 ceph-mon[12201]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:19:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:13 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:13 smithi070 ceph-mon[17138]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:19:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:13 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:14.456 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:14 smithi102 ceph-mon[13158]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:14.616 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:14 smithi070 ceph-mon[12201]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:14.616 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:14 smithi070 ceph-mon[17138]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:16.102 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:16.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:16 smithi070 ceph-mon[17138]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:16.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:16 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:16.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:16 smithi070 ceph-mon[12201]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:16.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:16 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:16 smithi102 ceph-mon[13158]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:16.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:16 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:18 smithi102 ceph-mon[13158]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:18 smithi070 ceph-mon[12201]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:18 smithi070 ceph-mon[17138]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:18.848 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:18] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:19 smithi102 ceph-mon[13158]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:19 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:19 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:19.932 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[12201]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:19.932 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:19.932 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:19.932 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[17138]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:19.932 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:19.933 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:19 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:22.388 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:22 smithi070 ceph-mon[12201]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:22.388 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:22 smithi070 ceph-mon[17138]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:22 smithi102 ceph-mon[13158]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:23.452 INFO:teuthology.orchestra.run.smithi070.stdout:34359738415 2024-08-31T02:19:23.452 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-31T02:19:24.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:24 smithi070 ceph-mon[12201]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:24.321 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:24 smithi070 ceph-mon[17138]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:24 smithi102 ceph-mon[13158]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:24.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:19:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:24.918 INFO:teuthology.orchestra.run.smithi070.stdout:133143986204 2024-08-31T02:19:24.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-31T02:19:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:25 smithi102 ceph-mon[13158]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:25 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:25 smithi070 ceph-mon[12201]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:25.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:25 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:25 smithi070 ceph-mon[17138]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:25 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:26.494 INFO:teuthology.orchestra.run.smithi070.stdout:111669149730 2024-08-31T02:19:26.494 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-31T02:19:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:27 smithi102 ceph-mon[13158]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:27 smithi070 ceph-mon[12201]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:27 smithi070 ceph-mon[17138]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:28.035 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:29.102 INFO:teuthology.orchestra.run.smithi070.stdout:201863462931 2024-08-31T02:19:29.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-31T02:19:29.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:29 smithi070 ceph-mon[12201]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:29.350 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:29 smithi070 ceph-mon[17138]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:29 smithi102 ceph-mon[13158]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:30.644 INFO:teuthology.orchestra.run.smithi070.stdout:236223201294 2024-08-31T02:19:30.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-31T02:19:31.599 INFO:teuthology.orchestra.run.smithi070.stdout:167503724568 2024-08-31T02:19:31.599 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-31T02:19:32.149 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:32.149 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:32.340 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:32 smithi070 ceph-mon[12201]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:32.341 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:32 smithi070 ceph-mon[17138]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:32.383 INFO:teuthology.orchestra.run.smithi070.stdout:55834574891 2024-08-31T02:19:32.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-31T02:19:32.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:32 smithi102 ceph-mon[13158]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:32.562 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:32.562 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:32.692 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:32 smithi070 systemd[1]: Starting Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:19:34.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:34 smithi070 ceph-mon[12201]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:34.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:34 smithi070 ceph-mon[17138]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:34.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:34 smithi102 ceph-mon[13158]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:36.038 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:36.134 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:36 smithi102 ceph-mon[13158]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:36 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:36 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[12201]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[17138]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:36 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:36.866 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:36.866 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:37.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:37 smithi070 podman[35746]: 2024-08-31 02:19:37.641881411 +0000 UTC m=+0.146195595 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-08-31T02:19:37.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:37 smithi070 podman[35746]: 2024-08-31 02:19:37.765548139 +0000 UTC m=+0.269862316 volume create 2a06674f323d0640fe35a5cce24d709d9021e363887bacb7a68394c04caa7db1 2024-08-31T02:19:37.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:37 smithi070 podman[35746]: 2024-08-31 02:19:37.849265619 +0000 UTC m=+0.353579796 container create faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:38.360 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:38 smithi070 ceph-mon[17138]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:38.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:38 smithi070 ceph-mon[12201]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:38 smithi102 ceph-mon[13158]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:38.777 INFO:teuthology.orchestra.run.smithi070.stdout:77309411369 2024-08-31T02:19:38.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-31T02:19:39.196 INFO:teuthology.orchestra.run.smithi070.stdout:111669149732 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 podman[35746]: 2024-08-31 02:19:39.015698855 +0000 UTC m=+1.520013030 container init faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 podman[35746]: 2024-08-31 02:19:39.020056355 +0000 UTC m=+1.524370532 container start faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.030Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.030Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.032Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.033Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.058Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:19:39.304 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.058Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:19:39.305 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.061Z caller=main.go:518 msg=Listening address=:9093 2024-08-31T02:19:39.305 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:39.061Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-08-31T02:19:39.305 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 bash[35746]: faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec 2024-08-31T02:19:39.305 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:39 smithi070 systemd[1]: Started Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3237348937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:40.185 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3237348937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:40.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:40 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3237348937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:40 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:41.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:41.033Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000113244s 2024-08-31T02:19:41.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:41 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:41.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:41 smithi070 ceph-mon[12201]: Deploying daemon grafana.a on smithi102 2024-08-31T02:19:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:41 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:41 smithi070 ceph-mon[17138]: Deploying daemon grafana.a on smithi102 2024-08-31T02:19:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:41 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:19:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:41 smithi102 ceph-mon[13158]: Deploying daemon grafana.a on smithi102 2024-08-31T02:19:42.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:42 smithi102 ceph-mon[13158]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:42.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:42 smithi070 ceph-mon[12201]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:42.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:42 smithi070 ceph-mon[17138]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:43.067 INFO:teuthology.orchestra.run.smithi070.stdout:201863462934 2024-08-31T02:19:43.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[12201]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:43.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:43.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:19:43.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1085816191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[17138]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:19:43.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:43 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1085816191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-31T02:19:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:43 smithi102 ceph-mon[13158]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:19:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:19:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:43 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1085816191' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-31T02:19:44.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:19:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:45.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149732 for osd.3 2024-08-31T02:19:45.211 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:45.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:45 smithi070 ceph-mon[12201]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:45.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:45 smithi070 ceph-mon[17138]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:45 smithi102 ceph-mon[13158]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:46.569 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462934 for osd.6 2024-08-31T02:19:46.569 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:47.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:19:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:47.972 INFO:teuthology.orchestra.run.smithi070.stdout:236223201298 2024-08-31T02:19:48.387 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:48 smithi102 ceph-mon[13158]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:48.387 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:48.387 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:48 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1380872719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[12201]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1380872719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[17138]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:48.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:48 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1380872719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-31T02:19:48.718 INFO:teuthology.orchestra.run.smithi070.stdout:133143986209 2024-08-31T02:19:48.957 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-08-31T02:19:48.958 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:49.321 INFO:teuthology.orchestra.run.smithi070.stdout:55834574895 2024-08-31T02:19:49.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:49 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1441245961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-31T02:19:49.322 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:49.034Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00185991s 2024-08-31T02:19:49.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:49 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1441245961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-31T02:19:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:49 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1441245961' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-31T02:19:49.988 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986209 for osd.4 2024-08-31T02:19:49.988 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:50.006 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:19:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:50 smithi102 ceph-mon[13158]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:50 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3299692375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-31T02:19:50.220 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:50 smithi070 ceph-mon[12201]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:50.220 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:50 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3299692375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-31T02:19:50.220 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:50 smithi070 ceph-mon[17138]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:50.220 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:50 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3299692375' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-31T02:19:50.712 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574891 got 55834574895 for osd.1 2024-08-31T02:19:50.712 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:51.168 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:50 smithi102 systemd[1]: Starting Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:19:51.446 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 podman[31384]: 2024-08-31 02:19:51.096533545 +0000 UTC m=+0.015075551 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-08-31T02:19:51.446 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 podman[31384]: 2024-08-31 02:19:51.226835578 +0000 UTC m=+0.145377631 container create 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, release=236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, version=8.5, vendor=Red Hat, Inc.) 2024-08-31T02:19:51.446 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 podman[31384]: 2024-08-31 02:19:51.445664797 +0000 UTC m=+0.364206800 container init 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8, release=236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, version=8.5, distribution-scope=public, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.buildah.version=1.24.2, vendor=Red Hat, Inc., maintainer=Paul Cuzner , name=ubi8) 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 podman[31384]: 2024-08-31 02:19:51.448614915 +0000 UTC m=+0.367156912 container start 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, maintainer=Paul Cuzner , io.openshift.expose-services=, release=236.1648460182, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, version=8.5, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vcs-type=git) 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-08-31T02:19:51.697 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-08-31T02:19:51.698 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-08-31T02:19:51.698 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-08-31T02:19:51.698 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-08-31T02:19:51.698 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="App mode production" logger=settings 2024-08-31T02:19:51.698 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-08-31T02:19:51.699 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-08-31T02:19:51.700 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-08-31T02:19:51.701 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-08-31T02:19:51.702 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-31T02:19:51.703 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-08-31T02:19:51.705 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-08-31T02:19:51.706 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 bash[31384]: 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 systemd[1]: Started Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-08-31T02:19:51.707 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-08-31T02:19:51.708 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-08-31T02:19:51.709 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-08-31T02:19:51.710 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-08-31T02:19:51.711 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-08-31T02:19:51.712 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-08-31T02:19:51.713 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-08-31T02:19:51.714 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-08-31T02:19:51.715 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-08-31T02:19:51.791 INFO:teuthology.orchestra.run.smithi070.stdout:167503724573 2024-08-31T02:19:51.947 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-08-31T02:19:51.948 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-08-31T02:19:51.949 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-08-31T02:19:51.950 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-08-31T02:19:51.951 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-08-31T02:19:51.952 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-08-31T02:19:51.952 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-08-31T02:19:51.952 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-08-31T02:19:51.952 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-08-31T02:19:51.952 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-08-31T02:19:51.953 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-31T02:19:51.954 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-08-31T02:19:51.955 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-31T02:19:51.956 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-31T02:19:51.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-08-31T02:19:51.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-08-31T02:19:51.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-08-31T02:19:51.960 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-08-31T02:19:51.960 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-08-31T02:19:51.960 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-08-31T02:19:51.960 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-08-31T02:19:51.960 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-08-31T02:19:51.961 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-08-31T02:19:51.962 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-31T02:19:51.963 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-08-31T02:19:51.964 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-08-31T02:19:51.965 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-08-31T02:19:51.976 INFO:teuthology.orchestra.run.smithi070.stdout:34359738421 2024-08-31T02:19:52.198 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.199 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3363609065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-31T02:19:52.200 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2044990188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-31T02:19:52.200 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-08-31T02:19:52.200 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-08-31T02:19:52.201 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-08-31T02:19:52.202 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:51+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-08-31T02:19:52.203 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=525.492013ms 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-08-31T02:19:52.204 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:19:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:19:52+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-08-31T02:19:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3363609065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-31T02:19:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2044990188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3363609065' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-31T02:19:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:52 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2044990188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-31T02:19:52.694 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724568 got 167503724573 for osd.5 2024-08-31T02:19:52.694 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:53.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738421 for osd.0 2024-08-31T02:19:53.142 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:53.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:53 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:19:53.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:53 smithi102 ceph-mon[13158]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:19:53.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:53 smithi102 ceph-mon[13158]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:19:53.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:53.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:19:53.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[12201]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[12201]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[17138]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[17138]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:19:53.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:54.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:54 smithi070 ceph-mon[12201]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:54.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:54 smithi070 ceph-mon[17138]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:54.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:54 smithi102 ceph-mon[13158]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:55.891 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:56.191 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:56 smithi070 ceph-mon[12201]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:56.191 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:56 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:56.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:56 smithi070 ceph-mon[17138]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:56.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:56 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:56.236 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:56 smithi102 ceph-mon[13158]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:56.237 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:56 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:56.655 INFO:teuthology.orchestra.run.smithi070.stdout:77309411373 2024-08-31T02:19:57.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:57 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2091996552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-31T02:19:57.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:57 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2091996552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-31T02:19:57.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:57 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2091996552' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-31T02:19:58.132 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[12201]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:58.132 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.132 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.132 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:19:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:19:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:19:58.133 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[17138]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:58.133 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.133 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:58 smithi102 ceph-mon[13158]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:19:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:19:58 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:19:58.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:58 smithi070 systemd[1]: Stopping Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:19:58.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:58 smithi070 bash[36870]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager.a" found: no such container 2024-08-31T02:19:58.571 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411369 got 77309411373 for osd.2 2024-08-31T02:19:58.572 DEBUG:teuthology.parallel:result is None 2024-08-31T02:19:58.572 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-31T02:19:58.572 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-31T02:19:58.895 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[35786]: level=info ts=2024-08-31T02:19:58.694Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:19:59.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:59 smithi070 podman[36898]: 2024-08-31 02:19:59.374441226 +0000 UTC m=+1.126701487 container died faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:59.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:59 smithi070 podman[36898]: 2024-08-31 02:19:59.840501302 +0000 UTC m=+1.592761551 container cleanup faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:19:59.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:19:59 smithi070 bash[36898]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-31T02:20:00.176 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:20:00.301 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:00 smithi070 ceph-mon[12201]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:00.302 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:00 smithi070 ceph-mon[12201]: overall HEALTH_OK 2024-08-31T02:20:00.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:00 smithi070 ceph-mon[17138]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:00.302 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:00 smithi070 ceph-mon[17138]: overall HEALTH_OK 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 podman[36945]: 2024-08-31 02:20:00.025230944 +0000 UTC m=+1.282812761 container remove faa20a4edd59e15fb7dc6f9f6e9bf8bb101c8bb17843c50a8496e104915152ec (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 podman[36945]: 2024-08-31 02:20:00.12621543 +0000 UTC m=+1.383797246 volume remove 2a06674f323d0640fe35a5cce24d709d9021e363887bacb7a68394c04caa7db1 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 bash[37021]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager.a" found: no such container 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 systemd[1]: Stopped Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:20:00.303 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:00 smithi070 systemd[1]: Starting Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:20:00.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:00 smithi102 ceph-mon[13158]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:00.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:00 smithi102 ceph-mon[13158]: overall HEALTH_OK 2024-08-31T02:20:01.926 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:01.926 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:01 smithi070 podman[37132]: 2024-08-31 02:20:01.501388039 +0000 UTC m=+0.019793066 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-08-31T02:20:01.926 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:01 smithi070 podman[37132]: 2024-08-31 02:20:01.618674159 +0000 UTC m=+0.137079215 volume create 3415c6c9a88c3a814c09d95f36535864b62ac780b8453a38279b664ce4f64713 2024-08-31T02:20:01.926 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:01 smithi070 podman[37132]: 2024-08-31 02:20:01.710736643 +0000 UTC m=+0.229141700 container create 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:02.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-mon[12201]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:02.298 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:02.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-mon[17138]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:02.299 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 podman[37132]: 2024-08-31 02:20:02.046779379 +0000 UTC m=+0.565184392 container init 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 podman[37132]: 2024-08-31 02:20:02.049790205 +0000 UTC m=+0.568195237 container start 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.059Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.059Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.060Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.062Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.088Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:20:02.299 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.088Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:20:02.300 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.090Z caller=main.go:518 msg=Listening address=:9093 2024-08-31T02:20:02.300 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:02.090Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-08-31T02:20:02.300 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 bash[37132]: 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 2024-08-31T02:20:02.300 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:02 smithi070 systemd[1]: Started Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:20:02.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:02 smithi102 ceph-mon[13158]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:02.857 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:20:02.859 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-08-31T02:20:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[12201]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:20:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[12201]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:20:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:20:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:03 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:20:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:03 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:03 smithi102 ceph-mon[13158]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:20:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:03 smithi102 ceph-mon[13158]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:20:03.675 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-08-31T02:20:02.296711+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48908,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912308,"statfs":{"total":767960285184,"available":767910203392,"internally_reserved":0,"allocated":6303744,"data_stored":3511269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002406"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-08-31T02:18:42.365068+0000","last_change":"2024-08-31T02:18:26.043143+0000","last_active":"2024-08-31T02:18:42.365068+0000","last_peered":"2024-08-31T02:18:42.365068+0000","last_clean":"2024-08-31T02:18:42.365068+0000","last_became_active":"2024-08-31T02:18:26.042686+0000","last_became_peered":"2024-08-31T02:18:26.042686+0000","last_unstale":"2024-08-31T02:18:42.365068+0000","last_undegraded":"2024-08-31T02:18:42.365068+0000","last_fullsized":"2024-08-31T02:18:42.365068+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T02:16:26.008516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T02:16:26.008516+0000","last_clean_scrub_stamp":"2024-08-31T02:16:26.008516+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T11:41:44.628981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.82299999999999995}]},{"osd":1,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.71299999999999997}]},{"osd":2,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.752},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.73399999999999999}]},{"osd":4,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.83299999999999996}]},{"osd":6,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.81000000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.442},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.434}]},{"osd":4,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.30099999999999999}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.374}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:05 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.315},"min":{"1min":0.23000000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.34599999999999997,"15min":0.33100000000000002},"min":{"1min":0.22,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Aug 31 02:19:31 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35799999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.379},{"interface":"front","average":{"1min":0.375,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Aug 31 02:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55700000000000005,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.432,"15min":0.442},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55800000000000005,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.307}]},{"osd":4,"last update":"Sat Aug 31 02:19:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.437}]},{"osd":5,"last update":"Sat Aug 31 02:19:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54600000000000004,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.65400000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39900000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.71299999999999997,"5min":0.747,"15min":0.747},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Aug 31 02:19:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.505},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.40799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 31 02:19:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34200000000000003},"min":{"1min":0.20699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.307},{"interface":"front","average":{"1min":0.378,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Aug 31 02:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Aug 31 02:19:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.375,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.54000000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.376}]},{"osd":4,"last update":"Sat Aug 31 02:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.499},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Aug 31 02:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.42899999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Aug 31 02:19:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.435}]},{"osd":7,"last update":"Sat Aug 31 02:19:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.61499999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36299999999999999,"15min":0.36699999999999999},"min":{"1min":0.249,"5min":0.187,"15min":0.187},"max":{"1min":0.51200000000000001,"5min":0.629,"15min":0.629},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.378,"15min":0.38200000000000001},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.48199999999999998,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Aug 31 02:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.37},"min":{"1min":0.23899999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.58599999999999997,"5min":0.623,"15min":0.623},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Aug 31 02:18:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.41799999999999998,"15min":0.41899999999999998},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53700000000000003,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.374},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.443,"15min":0.44600000000000001},"min":{"1min":0.28199999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.56200000000000006,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.37}]},{"osd":4,"last update":"Sat Aug 31 02:19:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68600000000000005,"5min":0.879,"15min":0.879},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.372,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Aug 31 02:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.438},{"interface":"front","average":{"1min":0.625,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.41999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Aug 31 02:18:57 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.504},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.53900000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39600000000000002,"15min":0.38900000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.373},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.28799999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41199999999999998,"15min":0.42199999999999999},"min":{"1min":0.27300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45000000000000001,"15min":0.45700000000000002},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.627,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.48899999999999999},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Aug 31 02:19:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.751,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Aug 31 02:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57999999999999996,"15min":0.57899999999999996},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77400000000000002,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Aug 31 02:19:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.66900000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57899999999999996,"15min":0.58399999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77800000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.68400000000000005,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.501}]},{"osd":1,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.34200000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69399999999999995,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39000000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73799999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.43099999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.65700000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48299999999999998}]},{"osd":3,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.54100000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69599999999999995,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.621},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55200000000000005,"15min":0.55100000000000005},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.72799999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.42899999999999999,"15min":0.42099999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81599999999999995,"5min":0.876,"15min":0.876},"last":0.32800000000000001}]},{"osd":6,"last update":"Sat Aug 31 02:18:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.377},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.35399999999999998}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.36499999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5636,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739516,"statfs":{"total":95995035648,"available":95989264384,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71599999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.52000000000000002,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.35899999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65300000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.495}]},{"osd":2,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.88,"15min":0.88},"last":0.45300000000000001}]},{"osd":3,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.377},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.57599999999999996,"5min":0.75,"15min":0.75},"last":0.443},{"interface":"front","average":{"1min":0.443,"5min":0.42299999999999999,"15min":0.41999999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.435}]},{"osd":6,"last update":"Sat Aug 31 02:18:57 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40100000000000002}]},{"osd":7,"last update":"Sat Aug 31 02:19:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.48399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T02:20:03.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph pg dump --format=json 2024-08-31T02:20:03.944 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:20:04.363 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:04 smithi070 ceph-mon[12201]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:04.363 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:04 smithi070 ceph-mon[12201]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:04.364 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:04.062Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000093454s 2024-08-31T02:20:04.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:04 smithi070 ceph-mon[17138]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:04.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:04 smithi070 ceph-mon[17138]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:04 smithi102 ceph-mon[13158]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:04 smithi102 ceph-mon[13158]: from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:06.388 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:06 smithi070 ceph-mon[17138]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:06.388 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:06 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:06.389 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:06 smithi070 ceph-mon[12201]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:06.389 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:06 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:06.475 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:06 smithi102 ceph-mon[13158]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:06.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:06 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:07.655 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:20:07.655 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 bash[32209]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:20:07.813 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:20:07.815 INFO:teuthology.orchestra.run.smithi070.stderr:dumped all 2024-08-31T02:20:07.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.654Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:20:07.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:20:07.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:20:07.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[29409]: ts=2024-08-31T02:20:07.655Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:20:07.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:07 smithi102 podman[32220]: 2024-08-31 02:20:07.66513216 +0000 UTC m=+0.147279287 container died 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:08.315 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:08 smithi070 ceph-mon[12201]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:08.316 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:08 smithi070 ceph-mon[17138]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:08.406 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-mon[13158]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:08.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32220]: 2024-08-31 02:20:08.121111281 +0000 UTC m=+0.603258441 container cleanup 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:08.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 bash[32220]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:20:08.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 bash[32248]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32233]: 2024-08-31 02:20:08.405797032 +0000 UTC m=+0.737118046 container remove 65456f77ac8aa05c933e09887933604a331cd11cc4bba09eb4dee20f92f0b163 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32311]: 2024-08-31 02:20:08.536799341 +0000 UTC m=+0.017216308 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:20:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32311]: 2024-08-31 02:20:08.648304028 +0000 UTC m=+0.128721028 container create 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:09.049 INFO:teuthology.orchestra.run.smithi070.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-08-31T02:20:06.297446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48908,"kb_used_data":6156,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912308,"statfs":{"total":767960285184,"available":767910203392,"internally_reserved":0,"allocated":6303744,"data_stored":3511269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002289"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-08-31T02:18:42.365068+0000","last_change":"2024-08-31T02:18:26.043143+0000","last_active":"2024-08-31T02:18:42.365068+0000","last_peered":"2024-08-31T02:18:42.365068+0000","last_clean":"2024-08-31T02:18:42.365068+0000","last_became_active":"2024-08-31T02:18:26.042686+0000","last_became_peered":"2024-08-31T02:18:26.042686+0000","last_unstale":"2024-08-31T02:18:42.365068+0000","last_undegraded":"2024-08-31T02:18:42.365068+0000","last_fullsized":"2024-08-31T02:18:42.365068+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T02:16:26.008516+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T02:16:26.008516+0000","last_clean_scrub_stamp":"2024-08-31T02:16:26.008516+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T11:41:44.628981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.625}]},{"osd":5,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.67400000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6156,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738996,"statfs":{"total":95995035648,"available":95988731904,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.52000000000000002,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.50900000000000001,"15min":0.50600000000000001},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.68600000000000005}]},{"osd":1,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.505,"15min":0.501},"min":{"1min":0.38500000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.76500000000000001}]},{"osd":2,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53100000000000003,"15min":0.53200000000000003},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.72699999999999998}]},{"osd":3,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.52800000000000002,"15min":0.52200000000000002},"min":{"1min":0.40000000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.71199999999999997}]},{"osd":4,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.44,"15min":0.442},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.61399999999999999,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.503},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42799999999999999,"15min":0.42399999999999999},"min":{"1min":0.33300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.437,"15min":0.434},"min":{"1min":0.33300000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.44500000000000001,"15min":0.437},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.629}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.56999999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:05 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.315},"min":{"1min":0.23000000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.34599999999999997,"15min":0.33100000000000002},"min":{"1min":0.22,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Aug 31 02:19:31 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.35799999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.379},{"interface":"front","average":{"1min":0.375,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Aug 31 02:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.55700000000000005,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.432,"15min":0.442},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.55800000000000005,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.307}]},{"osd":4,"last update":"Sat Aug 31 02:19:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38600000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.437}]},{"osd":5,"last update":"Sat Aug 31 02:19:38 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.54600000000000004,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.65400000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.56399999999999995,"15min":0.56799999999999995},"min":{"1min":0.39900000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.71299999999999997,"5min":0.747,"15min":0.747},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Aug 31 02:19:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.505},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.40799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 31 02:19:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34200000000000003},"min":{"1min":0.20699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.371,"15min":0.36399999999999999},"min":{"1min":0.22800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.318}]},{"osd":2,"last update":"Sat Aug 31 02:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Aug 31 02:19:53 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.312,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.375,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.54000000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.309}]},{"osd":4,"last update":"Sat Aug 31 02:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Aug 31 02:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55400000000000005,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57299999999999995,"15min":0.57399999999999995},"min":{"1min":0.42899999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.71799999999999997,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.439}]},{"osd":6,"last update":"Sat Aug 31 02:20:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57799999999999996,"15min":0.57899999999999996},"min":{"1min":0.41199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Aug 31 02:19:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36299999999999999,"15min":0.36699999999999999},"min":{"1min":0.249,"5min":0.187,"15min":0.187},"max":{"1min":0.51200000000000001,"5min":0.629,"15min":0.629},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.371,"5min":0.378,"15min":0.38200000000000001},"min":{"1min":0.22,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.48199999999999998,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33200000000000002}]},{"osd":1,"last update":"Sat Aug 31 02:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.36799999999999999,"15min":0.37},"min":{"1min":0.23899999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.58599999999999997,"5min":0.623,"15min":0.623},"last":0.312}]},{"osd":3,"last update":"Sat Aug 31 02:19:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.41099999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.51200000000000001,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.435,"15min":0.443},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.55300000000000005,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Aug 31 02:19:14 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55300000000000005},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68600000000000005,"5min":0.879,"15min":0.879},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.372,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Aug 31 02:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.58899999999999997,"15min":0.58299999999999996},"min":{"1min":0.41999999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56000000000000005}]},{"osd":6,"last update":"Sat Aug 31 02:19:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.41399999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.41999999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Aug 31 02:19:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.81200000000000006}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39600000000000002,"15min":0.38900000000000001},"min":{"1min":0.27400000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.28799999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41199999999999998,"15min":0.42199999999999999},"min":{"1min":0.27300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.435,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Aug 31 02:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45000000000000001,"15min":0.45700000000000002},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.627,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.48899999999999999},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Aug 31 02:19:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.70299999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.751,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Aug 31 02:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57999999999999996,"15min":0.57899999999999996},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77400000000000002,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.66500000000000004}]},{"osd":6,"last update":"Sat Aug 31 02:20:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58699999999999997,"15min":0.58299999999999996},"min":{"1min":0.42199999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.626,"15min":0.626},"min":{"1min":0.46000000000000002,"5min":0.433,"15min":0.433},"max":{"1min":0.81399999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.74099999999999999}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.68600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57899999999999996,"15min":0.58399999999999996},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77800000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.68400000000000005,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.434}]},{"osd":1,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.34200000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69399999999999995,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.443}]},{"osd":2,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39000000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73799999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.43099999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.65700000000000003,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Aug 31 02:19:17 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.54100000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69599999999999995,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55200000000000005,"15min":0.55100000000000005},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.72799999999999998,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.42899999999999999,"15min":0.42099999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81599999999999995,"5min":0.876,"15min":0.876},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Aug 31 02:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.48499999999999999,"15min":0.48199999999999998},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.77700000000000002,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46500000000000002,"15min":0.45700000000000002},"min":{"1min":0.33500000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Aug 31 02:19:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.34799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5636,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739516,"statfs":{"total":95995035648,"available":95989264384,"internally_reserved":0,"allocated":585728,"data_stored":240395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71599999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.52000000000000002,"15min":0.52100000000000002},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.53500000000000003}]},{"osd":1,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.35899999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.65300000000000002,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.58099999999999996}]},{"osd":2,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.88,"15min":0.88},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.55000000000000004,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68500000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Aug 31 02:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.57599999999999996,"5min":0.75,"15min":0.75},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.42299999999999999,"15min":0.41999999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Aug 31 02:20:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46600000000000003,"15min":0.46500000000000002},"min":{"1min":0.35099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.69299999999999995,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46500000000000002,"15min":0.46400000000000002},"min":{"1min":0.308,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Aug 31 02:19:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.67000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T02:20:09.050 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-31T02:20:09.051 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-31T02:20:09.051 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-31T02:20:09.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -- ceph health --format=json 2024-08-31T02:20:09.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32311]: 2024-08-31 02:20:08.816883161 +0000 UTC m=+0.297300116 container init 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 podman[32311]: 2024-08-31 02:20:08.819400024 +0000 UTC m=+0.299816979 container start 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.841Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.843Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.844Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.844Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.846Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:20:09.169 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.846Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.648µs 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.846Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.847Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.848Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.848Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.317µs wal_replay_duration=2.071243ms total_replay_duration=2.124177ms 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.850Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.850Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.850Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.866Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.880718ms db_storage=870ns remote_storage=1.232µs web_handler=314ns query_engine=597ns scrape=382.456µs scrape_sd=22.504µs notify=24.147µs notify_sd=6.258µs rules=15.220006ms 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:20:08.866Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 bash[32311]: 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 2024-08-31T02:20:09.170 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:20:08 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:20:09.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.521 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/mon.a/config 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:10 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.70:9093"}]: dispatch 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.102:3000"}]: dispatch 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:10 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:12.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:12 smithi070 ceph-mon[12201]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:12.285 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:20:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:20:12.063Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001846661s 2024-08-31T02:20:12.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:12 smithi070 ceph-mon[17138]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:12.296 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:20:12.297 INFO:teuthology.orchestra.run.smithi070.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-31T02:20:12.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:12 smithi102 ceph-mon[13158]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:13 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/979395114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-31T02:20:13.605 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:13 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/979395114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-31T02:20:13.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:13 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/979395114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-31T02:20:13.606 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:13.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:14 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:14.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:14.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:14 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:15.249 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-31T02:20:15.250 INFO:tasks.cephadm:Setup complete, yielding 2024-08-31T02:20:15.250 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-31T02:20:15.260 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi070.front.sepia.ceph.com 2024-08-31T02:20:15.260 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-08-31T02:20:16.396 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:16 smithi102 ceph-mon[13158]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:16 smithi070 ceph-mon[12201]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:16 smithi070 ceph-mon[17138]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:18.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:17 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:20:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:18.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:18 smithi102 ceph-mon[13158]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:18.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:18 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:18 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:20:18] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[17138]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[12201]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:18 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:20:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:20:19.244 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:20:19.245 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "c24d103f-f104-4880-8183-e965351bf3a6", 2024-08-31T02:20:19.245 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "r", 2024-08-31T02:20:19.245 INFO:teuthology.orchestra.run.smithi070.stdout: "current_period": "39930864-2119-4e75-82e2-7638f46da1a5", 2024-08-31T02:20:19.245 INFO:teuthology.orchestra.run.smithi070.stdout: "epoch": 1 2024-08-31T02:20:19.245 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:20:19.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[12201]: osdmap e61: 8 total, 8 up, 8 in 2024-08-31T02:20:19.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2122414287' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[17138]: osdmap e61: 8 total, 8 up, 8 in 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2122414287' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:19 smithi102 ceph-mon[13158]: osdmap e61: 8 total, 8 up, 8 in 2024-08-31T02:20:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:19 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2122414287' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:19 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-08-31T02:20:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:19 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:19 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:19.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-08-31T02:20:20.191 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:19] "GET /metrics HTTP/1.1" 200 191495 "" "Prometheus/2.33.4" 2024-08-31T02:20:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[12201]: pgmap v52: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-08-31T02:20:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[12201]: osdmap e62: 8 total, 8 up, 8 in 2024-08-31T02:20:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[17138]: pgmap v52: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-08-31T02:20:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:20 smithi070 ceph-mon[17138]: osdmap e62: 8 total, 8 up, 8 in 2024-08-31T02:20:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:20 smithi102 ceph-mon[13158]: pgmap v52: 33 pgs: 3 creating+peering, 1 active+clean, 29 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:20 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-08-31T02:20:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:20 smithi102 ceph-mon[13158]: osdmap e62: 8 total, 8 up, 8 in 2024-08-31T02:20:21.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:21 smithi070 ceph-mon[12201]: osdmap e63: 8 total, 8 up, 8 in 2024-08-31T02:20:21.541 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:21 smithi070 ceph-mon[17138]: osdmap e63: 8 total, 8 up, 8 in 2024-08-31T02:20:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:21 smithi102 ceph-mon[13158]: osdmap e63: 8 total, 8 up, 8 in 2024-08-31T02:20:21.756 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:20:21.756 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "2203cc4c-12a6-4bb5-a1df-e666ac0d6adc", 2024-08-31T02:20:21.756 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "default", 2024-08-31T02:20:21.756 INFO:teuthology.orchestra.run.smithi070.stdout: "api_name": "default", 2024-08-31T02:20:21.756 INFO:teuthology.orchestra.run.smithi070.stdout: "is_master": "true", 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "endpoints": [], 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "hostnames": [], 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "hostnames_s3website": [], 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "master_zone": "", 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "zones": [], 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "placement_targets": [], 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "default_placement": "", 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_id": "c24d103f-f104-4880-8183-e965351bf3a6", 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "sync_policy": { 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: "groups": [] 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:21.757 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:20:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:22 smithi070 ceph-mon[12201]: pgmap v55: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:22 smithi070 ceph-mon[17138]: pgmap v55: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:22.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-08-31T02:20:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:22 smithi102 ceph-mon[13158]: pgmap v55: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:24.380 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:24 smithi070 ceph-mon[12201]: pgmap v56: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:24.380 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:24 smithi070 ceph-mon[17138]: pgmap v56: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:24.470 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:20:24.470 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "3cf9f942-f6a7-444b-9b3c-05eb99eb296f", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "z", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "domain_root": "z.rgw.meta:root", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "control_pool": "z.rgw.control", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "gc_pool": "z.rgw.log:gc", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "lc_pool": "z.rgw.log:lc", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "log_pool": "z.rgw.log", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "roles_pool": "z.rgw.meta:roles", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-08-31T02:20:24.471 INFO:teuthology.orchestra.run.smithi070.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "otp_pool": "z.rgw.otp", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "system_key": { 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "access_key": "", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "secret_key": "" 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "placement_pools": [ 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "key": "default-placement", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "val": { 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "index_pool": "z.rgw.buckets.index", 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "storage_classes": { 2024-08-31T02:20:24.472 INFO:teuthology.orchestra.run.smithi070.stdout: "STANDARD": { 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: "data_pool": "z.rgw.buckets.data" 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: "index_type": 0 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_id": "c24d103f-f104-4880-8183-e965351bf3a6", 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout: "notif_pool": "z.rgw.log:notif" 2024-08-31T02:20:24.473 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:20:24.492 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:24 smithi102 ceph-mon[13158]: pgmap v56: 33 pgs: 1 active+clean, 32 creating+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:20:25.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-08-31T02:20:26.591 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:26 smithi070 ceph-mon[12201]: pgmap v57: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-08-31T02:20:26.591 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:26 smithi070 ceph-mon[17138]: pgmap v57: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-08-31T02:20:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:26 smithi102 ceph-mon[13158]: pgmap v57: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 672 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-08-31T02:20:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:27 smithi102 ceph-mon[13158]: pgmap v58: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:27 smithi070 ceph-mon[12201]: pgmap v58: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:27 smithi070 ceph-mon[17138]: pgmap v58: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:28.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[12201]: osdmap e64: 8 total, 8 up, 8 in 2024-08-31T02:20:28.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-08-31T02:20:28.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-08-31T02:20:28.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[12201]: osdmap e65: 8 total, 8 up, 8 in 2024-08-31T02:20:28.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[17138]: osdmap e64: 8 total, 8 up, 8 in 2024-08-31T02:20:28.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-08-31T02:20:28.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-08-31T02:20:28.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:28 smithi070 ceph-mon[17138]: osdmap e65: 8 total, 8 up, 8 in 2024-08-31T02:20:28.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:20:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:20:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:28 smithi102 ceph-mon[13158]: osdmap e64: 8 total, 8 up, 8 in 2024-08-31T02:20:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:28 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-08-31T02:20:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:28 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-08-31T02:20:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:28 smithi102 ceph-mon[13158]: osdmap e65: 8 total, 8 up, 8 in 2024-08-31T02:20:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:29 smithi102 ceph-mon[13158]: pgmap v60: 65 pgs: 6 creating+peering, 26 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 507 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:29 smithi102 ceph-mon[13158]: osdmap e66: 8 total, 8 up, 8 in 2024-08-31T02:20:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:29 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-08-31T02:20:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[12201]: pgmap v60: 65 pgs: 6 creating+peering, 26 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 507 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[12201]: osdmap e66: 8 total, 8 up, 8 in 2024-08-31T02:20:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-08-31T02:20:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[17138]: pgmap v60: 65 pgs: 6 creating+peering, 26 unknown, 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 507 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-08-31T02:20:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[17138]: osdmap e66: 8 total, 8 up, 8 in 2024-08-31T02:20:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-08-31T02:20:30.173 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:29] "GET /metrics HTTP/1.1" 200 193950 "" "Prometheus/2.33.4" 2024-08-31T02:20:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:31 smithi102 ceph-mon[13158]: pgmap v63: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-08-31T02:20:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:31 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-08-31T02:20:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:31 smithi102 ceph-mon[13158]: osdmap e67: 8 total, 8 up, 8 in 2024-08-31T02:20:31.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[17138]: pgmap v63: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-08-31T02:20:31.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-08-31T02:20:31.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[17138]: osdmap e67: 8 total, 8 up, 8 in 2024-08-31T02:20:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[12201]: pgmap v63: 97 pgs: 32 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 853 B/s wr, 5 op/s 2024-08-31T02:20:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1031842884' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-08-31T02:20:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:31 smithi070 ceph-mon[12201]: osdmap e67: 8 total, 8 up, 8 in 2024-08-31T02:20:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:32 smithi102 ceph-mon[13158]: osdmap e68: 8 total, 8 up, 8 in 2024-08-31T02:20:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:32 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-08-31T02:20:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:32 smithi070 ceph-mon[12201]: osdmap e68: 8 total, 8 up, 8 in 2024-08-31T02:20:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:32 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-08-31T02:20:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:32 smithi070 ceph-mon[17138]: osdmap e68: 8 total, 8 up, 8 in 2024-08-31T02:20:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:32 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-08-31T02:20:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:33 smithi102 ceph-mon[13158]: pgmap v66: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-08-31T02:20:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:33 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-08-31T02:20:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:33 smithi102 ceph-mon[13158]: osdmap e69: 8 total, 8 up, 8 in 2024-08-31T02:20:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:33 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-08-31T02:20:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[12201]: pgmap v66: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-08-31T02:20:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-08-31T02:20:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[12201]: osdmap e69: 8 total, 8 up, 8 in 2024-08-31T02:20:33.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-08-31T02:20:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[17138]: pgmap v66: 129 pgs: 64 unknown, 32 creating+peering, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-08-31T02:20:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-08-31T02:20:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[17138]: osdmap e69: 8 total, 8 up, 8 in 2024-08-31T02:20:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-08-31T02:20:34.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:33] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:34.406 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:20:34.406 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "1cb67ddc-eb8d-4a04-9328-582468e5337c", 2024-08-31T02:20:34.406 INFO:teuthology.orchestra.run.smithi070.stdout: "epoch": 1, 2024-08-31T02:20:34.406 INFO:teuthology.orchestra.run.smithi070.stdout: "predecessor_uuid": "39930864-2119-4e75-82e2-7638f46da1a5", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "sync_status": [], 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "period_map": { 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "1cb67ddc-eb8d-4a04-9328-582468e5337c", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "zonegroups": [ 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "2203cc4c-12a6-4bb5-a1df-e666ac0d6adc", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "default", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "api_name": "default", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "is_master": "true", 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "endpoints": [], 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "hostnames": [], 2024-08-31T02:20:34.407 INFO:teuthology.orchestra.run.smithi070.stdout: "hostnames_s3website": [], 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "master_zone": "3cf9f942-f6a7-444b-9b3c-05eb99eb296f", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "zones": [ 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "id": "3cf9f942-f6a7-444b-9b3c-05eb99eb296f", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "z", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "endpoints": [], 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "log_meta": "false", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "log_data": "false", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "bucket_index_max_shards": 11, 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "read_only": "false", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "tier_type": "", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "sync_from_all": "true", 2024-08-31T02:20:34.408 INFO:teuthology.orchestra.run.smithi070.stdout: "sync_from": [], 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "redirect_zone": "" 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "placement_targets": [ 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "name": "default-placement", 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "tags": [], 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "storage_classes": [ 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "STANDARD" 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:20:34.409 INFO:teuthology.orchestra.run.smithi070.stdout: "default_placement": "default-placement", 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_id": "c24d103f-f104-4880-8183-e965351bf3a6", 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "sync_policy": { 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "groups": [] 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "short_zone_ids": [ 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "key": "3cf9f942-f6a7-444b-9b3c-05eb99eb296f", 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: "val": 1526785102 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2024-08-31T02:20:34.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "master_zonegroup": "2203cc4c-12a6-4bb5-a1df-e666ac0d6adc", 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "master_zone": "3cf9f942-f6a7-444b-9b3c-05eb99eb296f", 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "period_config": { 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "bucket_quota": { 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "enabled": false, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "check_on_raw": false, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "max_size": -1, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "max_size_kb": 0, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "max_objects": -1 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "user_quota": { 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "enabled": false, 2024-08-31T02:20:34.411 INFO:teuthology.orchestra.run.smithi070.stdout: "check_on_raw": false, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_size": -1, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_size_kb": 0, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_objects": -1 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "user_ratelimit": { 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_ops": 0, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_ops": 0, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_bytes": 0, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_bytes": 0, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "enabled": false 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.412 INFO:teuthology.orchestra.run.smithi070.stdout: "bucket_ratelimit": { 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_ops": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_ops": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_bytes": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_bytes": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "enabled": false 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "anonymous_ratelimit": { 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_ops": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_ops": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_read_bytes": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "max_write_bytes": 0, 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: "enabled": false 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:20:34.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:20:34.414 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_id": "c24d103f-f104-4880-8183-e965351bf3a6", 2024-08-31T02:20:34.414 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_name": "r", 2024-08-31T02:20:34.414 INFO:teuthology.orchestra.run.smithi070.stdout: "realm_epoch": 2 2024-08-31T02:20:34.414 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:20:34.607 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:34 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-08-31T02:20:34.607 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:34 smithi102 ceph-mon[13158]: osdmap e70: 8 total, 8 up, 8 in 2024-08-31T02:20:34.607 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:34 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-08-31T02:20:34.608 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:34 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:34.608 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:34 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-08-31T02:20:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[12201]: osdmap e70: 8 total, 8 up, 8 in 2024-08-31T02:20:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-08-31T02:20:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-08-31T02:20:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[17138]: osdmap e70: 8 total, 8 up, 8 in 2024-08-31T02:20:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-08-31T02:20:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:34 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:35.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-08-31T02:20:35.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:35 smithi102 ceph-mon[13158]: pgmap v69: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-08-31T02:20:35.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:35 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-08-31T02:20:35.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:35 smithi102 ceph-mon[13158]: osdmap e71: 8 total, 8 up, 8 in 2024-08-31T02:20:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[12201]: pgmap v69: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-08-31T02:20:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-08-31T02:20:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[12201]: osdmap e71: 8 total, 8 up, 8 in 2024-08-31T02:20:35.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[17138]: pgmap v69: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-08-31T02:20:35.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/31530044' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-08-31T02:20:35.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-mon[17138]: osdmap e71: 8 total, 8 up, 8 in 2024-08-31T02:20:36.197 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:20:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:36.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:36 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:36.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:36 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:36.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:36 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:36.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:36 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:37.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:36 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:37.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:36 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:37.264 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled rgw.foo update... 2024-08-31T02:20:38.030 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph osd pool create foo' 2024-08-31T02:20:38.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: pgmap v71: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 412 B/s wr, 1 op/s 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:38.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:38.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: pgmap v71: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 412 B/s wr, 1 op/s 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:38.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: pgmap v71: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 206 B/s rd, 412 B/s wr, 1 op/s 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='client.14685 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:38.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:38.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:38 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:20:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:20:39.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:39 smithi070 ceph-mon[17138]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:39.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:39 smithi070 ceph-mon[17138]: Deploying daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:20:39.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:39 smithi070 ceph-mon[12201]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:39.535 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:39 smithi070 ceph-mon[12201]: Deploying daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:20:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:39 smithi102 ceph-mon[13158]: Saving service rgw.foo spec with placement count:2 2024-08-31T02:20:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:39 smithi102 ceph-mon[13158]: Deploying daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:20:40.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:39] "GET /metrics HTTP/1.1" 200 193950 "" "Prometheus/2.33.4" 2024-08-31T02:20:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:40 smithi102 ceph-mon[13158]: pgmap v72: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 682 B/s wr, 1 op/s 2024-08-31T02:20:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:40 smithi070 ceph-mon[12201]: pgmap v72: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 682 B/s wr, 1 op/s 2024-08-31T02:20:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:40 smithi070 ceph-mon[17138]: pgmap v72: 129 pgs: 129 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 682 B/s wr, 1 op/s 2024-08-31T02:20:41.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:41 smithi070 ceph-mon[12201]: pgmap v73: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-08-31T02:20:41.365 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:41 smithi070 ceph-mon[17138]: pgmap v73: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-08-31T02:20:41.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:41 smithi102 ceph-mon[13158]: pgmap v73: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-08-31T02:20:42.284 INFO:teuthology.orchestra.run.smithi070.stderr:pool 'foo' created 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1488372598' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:42.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:42 smithi102 ceph-mon[13158]: Deploying daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1488372598' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:42.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[12201]: Deploying daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1488372598' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:42.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:42 smithi070 ceph-mon[17138]: Deploying daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:20:42.876 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'rbd pool init foo' 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: osdmap e72: 8 total, 8 up, 8 in 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: pgmap v75: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:20:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:43 smithi102 ceph-mon[13158]: osdmap e73: 8 total, 8 up, 8 in 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: osdmap e72: 8 total, 8 up, 8 in 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: pgmap v75: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[12201]: osdmap e73: 8 total, 8 up, 8 in 2024-08-31T02:20:43.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-08-31T02:20:43.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: osdmap e72: 8 total, 8 up, 8 in 2024-08-31T02:20:43.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: pgmap v75: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-08-31T02:20:43.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:20:43.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:20:43.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:43 smithi070 ceph-mon[17138]: osdmap e73: 8 total, 8 up, 8 in 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: osdmap e74: 8 total, 8 up, 8 in 2024-08-31T02:20:45.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-08-31T02:20:45.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:45 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: osdmap e74: 8 total, 8 up, 8 in 2024-08-31T02:20:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: osdmap e74: 8 total, 8 up, 8 in 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-08-31T02:20:45.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:45 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:46.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 6.3 KiB/s wr, 247 op/s 2024-08-31T02:20:46.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:46.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[12201]: osdmap e75: 8 total, 8 up, 8 in 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 6.3 KiB/s wr, 247 op/s 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-08-31T02:20:46.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:46 smithi070 ceph-mon[17138]: osdmap e75: 8 total, 8 up, 8 in 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 6.3 KiB/s wr, 247 op/s 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: Checking dashboard <-> RGW credentials 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/650356752' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-08-31T02:20:46.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:46 smithi102 ceph-mon[13158]: osdmap e75: 8 total, 8 up, 8 in 2024-08-31T02:20:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[12201]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 166 KiB/s rd, 8.2 KiB/s wr, 366 op/s 2024-08-31T02:20:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[12201]: osdmap e76: 8 total, 8 up, 8 in 2024-08-31T02:20:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[17138]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 166 KiB/s rd, 8.2 KiB/s wr, 366 op/s 2024-08-31T02:20:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[17138]: osdmap e76: 8 total, 8 up, 8 in 2024-08-31T02:20:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:47.740 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:47 smithi102 ceph-mon[13158]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 166 KiB/s rd, 8.2 KiB/s wr, 366 op/s 2024-08-31T02:20:47.740 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:47 smithi102 ceph-mon[13158]: osdmap e76: 8 total, 8 up, 8 in 2024-08-31T02:20:47.740 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:47 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.194 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:20:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:48.195 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:48.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch apply iscsi foo u p' 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[12201]: osdmap e77: 8 total, 8 up, 8 in 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[17138]: osdmap e77: 8 total, 8 up, 8 in 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.841 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.845 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:48 smithi102 ceph-mon[13158]: osdmap e77: 8 total, 8 up, 8 in 2024-08-31T02:20:48.846 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.846 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:48.846 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:20:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:20:50.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:49 smithi070 ceph-mon[12201]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 249 KiB/s rd, 1.2 KiB/s wr, 372 op/s 2024-08-31T02:20:50.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:49] "GET /metrics HTTP/1.1" 200 203496 "" "Prometheus/2.33.4" 2024-08-31T02:20:50.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:20:50] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:50.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:49 smithi070 ceph-mon[17138]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 249 KiB/s rd, 1.2 KiB/s wr, 372 op/s 2024-08-31T02:20:50.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:49 smithi102 ceph-mon[13158]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 249 KiB/s rd, 1.2 KiB/s wr, 372 op/s 2024-08-31T02:20:50.653 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled iscsi.foo update... 2024-08-31T02:20:51.295 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.295 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:51.296 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:51.297 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:51.297 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:51 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:51.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:51 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:52 smithi102 ceph-mon[13158]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 853 B/s wr, 293 op/s 2024-08-31T02:20:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:52 smithi102 ceph-mon[13158]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:52 smithi102 ceph-mon[13158]: Saving service iscsi.foo spec with placement count:1 2024-08-31T02:20:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:52 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[12201]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 853 B/s wr, 293 op/s 2024-08-31T02:20:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[12201]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[12201]: Saving service iscsi.foo spec with placement count:1 2024-08-31T02:20:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[17138]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 853 B/s wr, 293 op/s 2024-08-31T02:20:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[17138]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:20:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[17138]: Saving service iscsi.foo spec with placement count:1 2024-08-31T02:20:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:52.729 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:20:53.044 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:20:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 753 B/s wr, 258 op/s 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: Checking pool "foo" exists for service iscsi.foo 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:53 smithi102 ceph-mon[13158]: Deploying daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:20:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 753 B/s wr, 258 op/s 2024-08-31T02:20:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: Checking pool "foo" exists for service iscsi.foo 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[12201]: Deploying daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:20:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 753 B/s wr, 258 op/s 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: Checking pool "foo" exists for service iscsi.foo 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:53.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:53 smithi070 ceph-mon[17138]: Deploying daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:20:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:55 smithi102 ceph-mon[13158]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 639 B/s wr, 234 op/s 2024-08-31T02:20:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:55 smithi102 ceph-mon[13158]: mgrmap e20: y(active, since 2m), standbys: x 2024-08-31T02:20:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:55 smithi102 ceph-mon[13158]: osdmap e78: 8 total, 8 up, 8 in 2024-08-31T02:20:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[12201]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 639 B/s wr, 234 op/s 2024-08-31T02:20:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[12201]: mgrmap e20: y(active, since 2m), standbys: x 2024-08-31T02:20:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[12201]: osdmap e78: 8 total, 8 up, 8 in 2024-08-31T02:20:55.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[17138]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 639 B/s wr, 234 op/s 2024-08-31T02:20:55.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[17138]: mgrmap e20: y(active, since 2m), standbys: x 2024-08-31T02:20:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:55 smithi070 ceph-mon[17138]: osdmap e78: 8 total, 8 up, 8 in 2024-08-31T02:20:57.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 582 B/s wr, 213 op/s 2024-08-31T02:20:57.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.656 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:57 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 582 B/s wr, 213 op/s 2024-08-31T02:20:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 70 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 582 B/s wr, 213 op/s 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:20:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:20:57.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:20:57.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:57.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:57 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: Adding iSCSI gateway http://:@172.21.15.70:5000 to Dashboard 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2069588656' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:20:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1557320351' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:58.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:58 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: Adding iSCSI gateway http://:@172.21.15.70:5000 to Dashboard 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2069588656' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:20:58.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1557320351' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: Adding iSCSI gateway http://:@172.21.15.70:5000 to Dashboard 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi070"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2069588656' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1557320351' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:58 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]: dispatch 2024-08-31T02:20:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[12201]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 102 B/s wr, 38 op/s 2024-08-31T02:20:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]': finished 2024-08-31T02:20:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[12201]: osdmap e79: 8 total, 8 up, 8 in 2024-08-31T02:20:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]: dispatch 2024-08-31T02:20:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[17138]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 102 B/s wr, 38 op/s 2024-08-31T02:20:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]': finished 2024-08-31T02:20:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[17138]: osdmap e79: 8 total, 8 up, 8 in 2024-08-31T02:20:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]: dispatch 2024-08-31T02:20:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:59 smithi102 ceph-mon[13158]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 102 B/s wr, 38 op/s 2024-08-31T02:20:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:59 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2952173293"}]': finished 2024-08-31T02:20:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:59 smithi102 ceph-mon[13158]: osdmap e79: 8 total, 8 up, 8 in 2024-08-31T02:20:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:20:59 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]: dispatch 2024-08-31T02:20:59.992 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:20:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:20:59] "GET /metrics HTTP/1.1" 200 208024 "" "Prometheus/2.33.4" 2024-08-31T02:21:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:00 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]': finished 2024-08-31T02:21:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:00 smithi102 ceph-mon[13158]: osdmap e80: 8 total, 8 up, 8 in 2024-08-31T02:21:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:00 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]: dispatch 2024-08-31T02:21:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]': finished 2024-08-31T02:21:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[12201]: osdmap e80: 8 total, 8 up, 8 in 2024-08-31T02:21:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]: dispatch 2024-08-31T02:21:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3676377318' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3561013195"}]': finished 2024-08-31T02:21:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[17138]: osdmap e80: 8 total, 8 up, 8 in 2024-08-31T02:21:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:00 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]: dispatch 2024-08-31T02:21:01.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:21:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:01.417 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[12201]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 1 op/s 2024-08-31T02:21:01.417 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]': finished 2024-08-31T02:21:01.417 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[12201]: osdmap e81: 8 total, 8 up, 8 in 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]: dispatch 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[17138]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 1 op/s 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]': finished 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[17138]: osdmap e81: 8 total, 8 up, 8 in 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]: dispatch 2024-08-31T02:21:01.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:01 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:01 smithi102 ceph-mon[13158]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 1 op/s 2024-08-31T02:21:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:01 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1926544973' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3189512648"}]': finished 2024-08-31T02:21:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:01 smithi102 ceph-mon[13158]: osdmap e81: 8 total, 8 up, 8 in 2024-08-31T02:21:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:01 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]: dispatch 2024-08-31T02:21:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:01 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:02 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]': finished 2024-08-31T02:21:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:02 smithi102 ceph-mon[13158]: osdmap e82: 8 total, 8 up, 8 in 2024-08-31T02:21:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:02 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3642062281' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:02 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]': finished 2024-08-31T02:21:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[12201]: osdmap e82: 8 total, 8 up, 8 in 2024-08-31T02:21:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3642062281' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/129248426' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3589050309"}]': finished 2024-08-31T02:21:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[17138]: osdmap e82: 8 total, 8 up, 8 in 2024-08-31T02:21:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3642062281' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:02 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]: dispatch 2024-08-31T02:21:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:03 smithi102 ceph-mon[13158]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:03 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]': finished 2024-08-31T02:21:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:03 smithi102 ceph-mon[13158]: osdmap e83: 8 total, 8 up, 8 in 2024-08-31T02:21:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:03 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]: dispatch 2024-08-31T02:21:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[12201]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]': finished 2024-08-31T02:21:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[12201]: osdmap e83: 8 total, 8 up, 8 in 2024-08-31T02:21:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]: dispatch 2024-08-31T02:21:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[17138]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 74 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/2881947222"}]': finished 2024-08-31T02:21:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[17138]: osdmap e83: 8 total, 8 up, 8 in 2024-08-31T02:21:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]: dispatch 2024-08-31T02:21:04.412 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:21:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]': finished 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: osdmap e84: 8 total, 8 up, 8 in 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]: dispatch 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:21:04.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:21:04.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]': finished 2024-08-31T02:21:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: osdmap e84: 8 total, 8 up, 8 in 2024-08-31T02:21:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]: dispatch 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/945893771' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3237384352"}]': finished 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: osdmap e84: 8 total, 8 up, 8 in 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]: dispatch 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:21:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:21:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:21:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:05.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:05 smithi102 ceph-mon[13158]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:05.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:05 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]': finished 2024-08-31T02:21:05.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:05 smithi102 ceph-mon[13158]: osdmap e85: 8 total, 8 up, 8 in 2024-08-31T02:21:05.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:05 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/129945891' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:05.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:05 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[12201]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]': finished 2024-08-31T02:21:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[12201]: osdmap e85: 8 total, 8 up, 8 in 2024-08-31T02:21:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/129945891' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[17138]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2429135725' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3252563261"}]': finished 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[17138]: osdmap e85: 8 total, 8 up, 8 in 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/129945891' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:05 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]: dispatch 2024-08-31T02:21:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:06 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]': finished 2024-08-31T02:21:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:06 smithi102 ceph-mon[13158]: osdmap e86: 8 total, 8 up, 8 in 2024-08-31T02:21:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:06 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2148980230' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:06 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]': finished 2024-08-31T02:21:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[12201]: osdmap e86: 8 total, 8 up, 8 in 2024-08-31T02:21:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2148980230' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/2881947222"}]': finished 2024-08-31T02:21:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[17138]: osdmap e86: 8 total, 8 up, 8 in 2024-08-31T02:21:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2148980230' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:06 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]: dispatch 2024-08-31T02:21:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:07 smithi102 ceph-mon[13158]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:07 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]': finished 2024-08-31T02:21:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:07 smithi102 ceph-mon[13158]: osdmap e87: 8 total, 8 up, 8 in 2024-08-31T02:21:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/15957909' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:07 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:07.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:21:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[12201]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]': finished 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[12201]: osdmap e87: 8 total, 8 up, 8 in 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/15957909' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[17138]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3252563261"}]': finished 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[17138]: osdmap e87: 8 total, 8 up, 8 in 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/15957909' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:07 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]: dispatch 2024-08-31T02:21:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]': finished 2024-08-31T02:21:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: osdmap e88: 8 total, 8 up, 8 in 2024-08-31T02:21:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]: dispatch 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]': finished 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: osdmap e88: 8 total, 8 up, 8 in 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:08 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]: dispatch 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1047345415"}]': finished 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: osdmap e88: 8 total, 8 up, 8 in 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:08 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]: dispatch 2024-08-31T02:21:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:09 smithi102 ceph-mon[13158]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]': finished 2024-08-31T02:21:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:09 smithi102 ceph-mon[13158]: osdmap e89: 8 total, 8 up, 8 in 2024-08-31T02:21:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4225784248' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:09 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[12201]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]': finished 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[12201]: osdmap e89: 8 total, 8 up, 8 in 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4225784248' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:09.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[17138]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 75 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1427071843' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2643865740"}]': finished 2024-08-31T02:21:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[17138]: osdmap e89: 8 total, 8 up, 8 in 2024-08-31T02:21:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4225784248' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]: dispatch 2024-08-31T02:21:10.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:09] "GET /metrics HTTP/1.1" 200 208024 "" "Prometheus/2.33.4" 2024-08-31T02:21:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:10 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]': finished 2024-08-31T02:21:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:10 smithi102 ceph-mon[13158]: osdmap e90: 8 total, 8 up, 8 in 2024-08-31T02:21:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:10 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1032042865' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:10 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]': finished 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[12201]: osdmap e90: 8 total, 8 up, 8 in 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1032042865' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/848058187"}]': finished 2024-08-31T02:21:10.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[17138]: osdmap e90: 8 total, 8 up, 8 in 2024-08-31T02:21:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1032042865' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:10 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]: dispatch 2024-08-31T02:21:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:11 smithi102 ceph-mon[13158]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:11 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]': finished 2024-08-31T02:21:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:11 smithi102 ceph-mon[13158]: osdmap e91: 8 total, 8 up, 8 in 2024-08-31T02:21:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:11 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2540566184' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:11 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:11.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[12201]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:11.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]': finished 2024-08-31T02:21:11.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[12201]: osdmap e91: 8 total, 8 up, 8 in 2024-08-31T02:21:11.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2540566184' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[17138]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3030506206"}]': finished 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[17138]: osdmap e91: 8 total, 8 up, 8 in 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2540566184' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:11.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:11 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]: dispatch 2024-08-31T02:21:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:12 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]': finished 2024-08-31T02:21:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:12 smithi102 ceph-mon[13158]: osdmap e92: 8 total, 8 up, 8 in 2024-08-31T02:21:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:12 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]: dispatch 2024-08-31T02:21:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]': finished 2024-08-31T02:21:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[12201]: osdmap e92: 8 total, 8 up, 8 in 2024-08-31T02:21:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]: dispatch 2024-08-31T02:21:12.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/589807710"}]': finished 2024-08-31T02:21:12.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[17138]: osdmap e92: 8 total, 8 up, 8 in 2024-08-31T02:21:12.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:12 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]: dispatch 2024-08-31T02:21:13.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:13 smithi102 ceph-mon[13158]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:13.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:13 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]': finished 2024-08-31T02:21:13.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:13 smithi102 ceph-mon[13158]: osdmap e93: 8 total, 8 up, 8 in 2024-08-31T02:21:13.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:13 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]: dispatch 2024-08-31T02:21:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[12201]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]': finished 2024-08-31T02:21:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[12201]: osdmap e93: 8 total, 8 up, 8 in 2024-08-31T02:21:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]: dispatch 2024-08-31T02:21:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[17138]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 79 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:21:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4207842621' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3374058595"}]': finished 2024-08-31T02:21:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[17138]: osdmap e93: 8 total, 8 up, 8 in 2024-08-31T02:21:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:13 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]: dispatch 2024-08-31T02:21:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:14 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]': finished 2024-08-31T02:21:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:14 smithi102 ceph-mon[13158]: osdmap e94: 8 total, 8 up, 8 in 2024-08-31T02:21:14.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:14 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]': finished 2024-08-31T02:21:14.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:14 smithi070 ceph-mon[17138]: osdmap e94: 8 total, 8 up, 8 in 2024-08-31T02:21:14.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:14 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/674700236' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3030506206"}]': finished 2024-08-31T02:21:14.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:14 smithi070 ceph-mon[12201]: osdmap e94: 8 total, 8 up, 8 in 2024-08-31T02:21:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:16 smithi102 ceph-mon[13158]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:16 smithi070 ceph-mon[12201]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:16 smithi070 ceph-mon[17138]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:18.333 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:18 smithi070 ceph-mon[12201]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:21:18.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:18 smithi070 ceph-mon[17138]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:21:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:18 smithi102 ceph-mon[13158]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 80 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:21:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:20.040 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:19] "GET /metrics HTTP/1.1" 200 208090 "" "Prometheus/2.33.4" 2024-08-31T02:21:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:20 smithi102 ceph-mon[13158]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:21:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:20 smithi070 ceph-mon[12201]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:21:20.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:21:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:20.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:20 smithi070 ceph-mon[17138]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:21:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:21 smithi102 ceph-mon[13158]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[12201]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:21 smithi070 ceph-mon[17138]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:23 smithi102 ceph-mon[13158]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:21:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:23 smithi070 ceph-mon[12201]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:21:23.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:23 smithi070 ceph-mon[17138]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:21:26.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:26 smithi102 ceph-mon[13158]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-08-31T02:21:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:26 smithi070 ceph-mon[12201]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-08-31T02:21:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:26 smithi070 ceph-mon[17138]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-08-31T02:21:27.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:21:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:28 smithi102 ceph-mon[13158]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:28 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:28 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[12201]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[17138]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:28.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:28 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:28.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:29.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:30.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:30 smithi102 ceph-mon[13158]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:30 smithi070 ceph-mon[12201]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:30.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:29] "GET /metrics HTTP/1.1" 200 208052 "" "Prometheus/2.33.4" 2024-08-31T02:21:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:30 smithi070 ceph-mon[17138]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:32 smithi102 ceph-mon[13158]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:32 smithi070 ceph-mon[12201]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:32 smithi070 ceph-mon[17138]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:34.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:34 smithi102 ceph-mon[13158]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:34 smithi070 ceph-mon[12201]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:34 smithi070 ceph-mon[17138]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:36 smithi102 ceph-mon[13158]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:36 smithi070 ceph-mon[12201]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:36 smithi070 ceph-mon[17138]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:38.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:38 smithi070 ceph-mon[12201]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:38.386 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:38 smithi070 ceph-mon[17138]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:38.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:38 smithi102 ceph-mon[13158]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:39.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:40.101 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:39] "GET /metrics HTTP/1.1" 200 208052 "" "Prometheus/2.33.4" 2024-08-31T02:21:40.378 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:40 smithi070 ceph-mon[12201]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:40.378 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:40 smithi070 ceph-mon[17138]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:40.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:40 smithi102 ceph-mon[13158]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:40.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:21:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:41.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[12201]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:41.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:41.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[17138]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:41 smithi102 ceph-mon[13158]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-08-31T02:21:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-08-31T02:21:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:21:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-08-31T02:21:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-08-31T02:21:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:21:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-08-31T02:21:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-08-31T02:21:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-08-31T02:21:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-31T02:21:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-31T02:21:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:43 smithi102 ceph-mon[13158]: osdmap e95: 8 total, 8 up, 8 in 2024-08-31T02:21:43.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[12201]: osdmap e95: 8 total, 8 up, 8 in 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-08-31T02:21:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-31T02:21:43.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-08-31T02:21:43.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-31T02:21:43.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:43 smithi070 ceph-mon[17138]: osdmap e95: 8 total, 8 up, 8 in 2024-08-31T02:21:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:44 smithi102 ceph-mon[13158]: osdmap e96: 8 total, 8 up, 8 in 2024-08-31T02:21:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:44 smithi070 ceph-mon[12201]: osdmap e96: 8 total, 8 up, 8 in 2024-08-31T02:21:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:44 smithi070 ceph-mon[17138]: osdmap e96: 8 total, 8 up, 8 in 2024-08-31T02:21:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:46 smithi102 ceph-mon[13158]: pgmap v130: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:46 smithi070 ceph-mon[12201]: pgmap v130: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:46.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:46 smithi070 ceph-mon[17138]: pgmap v130: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:21:47.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:21:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:21:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:48 smithi102 ceph-mon[13158]: pgmap v131: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:21:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[12201]: pgmap v131: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[17138]: pgmap v131: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:21:48.904 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:49.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:49.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:50.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:50 smithi102 ceph-mon[13158]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:50.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:50 smithi070 ceph-mon[12201]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:50.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:49] "GET /metrics HTTP/1.1" 200 208033 "" "Prometheus/2.33.4" 2024-08-31T02:21:50.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:50 smithi070 ceph-mon[17138]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:52 smithi102 ceph-mon[13158]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:52 smithi070 ceph-mon[12201]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:52.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:52 smithi070 ceph-mon[17138]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 5 B/s, 0 objects/s recovering 2024-08-31T02:21:54.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:54 smithi102 ceph-mon[13158]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:54 smithi070 ceph-mon[12201]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:54.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:54 smithi070 ceph-mon[17138]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:56.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:56 smithi102 ceph-mon[13158]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:56 smithi070 ceph-mon[12201]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:56.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:56 smithi070 ceph-mon[17138]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 936 B/s rd, 0 op/s; 4 B/s, 0 objects/s recovering 2024-08-31T02:21:58.409 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:58 smithi102 ceph-mon[13158]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:21:58.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:58 smithi070 ceph-mon[12201]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:21:58.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:58 smithi070 ceph-mon[17138]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:21:58.667 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:21:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:21:59.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:21:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:59.387 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:21:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:21:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:21:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:00.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:00 smithi070 ceph-mon[12201]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:22:00.414 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:21:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:21:59] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-08-31T02:22:00.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:00 smithi070 ceph-mon[17138]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:22:00.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:00 smithi102 ceph-mon[13158]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 3 B/s, 0 objects/s recovering 2024-08-31T02:22:00.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:22:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:01 smithi102 ceph-mon[13158]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:01 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:01 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:01.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[12201]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:01.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:01.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:01.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[17138]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:01 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:03 smithi102 ceph-mon[13158]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:03.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:03 smithi070 ceph-mon[12201]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:03.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:03 smithi070 ceph-mon[17138]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:04.850 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:22:04.850 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:22:04.851 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:22:04.851 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:22:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:22:04.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:06 smithi102 ceph-mon[13158]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:06.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:06 smithi070 ceph-mon[12201]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:06 smithi070 ceph-mon[17138]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:08.084 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:22:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:08 smithi102 ceph-mon[13158]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[17138]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:08.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[12201]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:08.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:09 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:09.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:09 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:09.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:09 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:10 smithi102 ceph-mon[13158]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:10 smithi070 ceph-mon[12201]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:10 smithi070 ceph-mon[17138]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:10.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:09] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-08-31T02:22:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:12 smithi102 ceph-mon[13158]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:12 smithi070 ceph-mon[12201]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:12.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:12 smithi070 ceph-mon[17138]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:14 smithi102 ceph-mon[13158]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:14.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:14 smithi070 ceph-mon[12201]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:14 smithi070 ceph-mon[17138]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:16 smithi102 ceph-mon[13158]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:16 smithi070 ceph-mon[12201]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:16.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:16 smithi070 ceph-mon[17138]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:18 smithi102 ceph-mon[13158]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:18 smithi070 ceph-mon[12201]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:18 smithi070 ceph-mon[17138]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:20.106 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:19] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-08-31T02:22:20.373 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:20 smithi070 ceph-mon[12201]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:20.373 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:20 smithi070 ceph-mon[17138]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:20.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:20 smithi102 ceph-mon[13158]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:20.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:22:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:21.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:21 smithi102 ceph-mon[13158]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:21.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:21.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[17138]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[12201]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:21.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:23.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:23 smithi102 ceph-mon[13158]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:23.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:23 smithi070 ceph-mon[12201]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:23 smithi070 ceph-mon[17138]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:26 smithi102 ceph-mon[13158]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:26 smithi070 ceph-mon[12201]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:26 smithi070 ceph-mon[17138]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:28.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[12201]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:22:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[17138]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:28 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:28 smithi102 ceph-mon[13158]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:28 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:28 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:28.761 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:29.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:30.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:30 smithi102 ceph-mon[13158]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:30 smithi070 ceph-mon[12201]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:30 smithi070 ceph-mon[17138]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:30.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:29] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-08-31T02:22:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:32 smithi102 ceph-mon[13158]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:32 smithi070 ceph-mon[12201]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:32 smithi070 ceph-mon[17138]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:34.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:34 smithi102 ceph-mon[13158]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:34 smithi070 ceph-mon[12201]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:34 smithi070 ceph-mon[17138]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:36 smithi102 ceph-mon[13158]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:36 smithi070 ceph-mon[12201]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:36 smithi070 ceph-mon[17138]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:38 smithi102 ceph-mon[13158]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:38 smithi070 ceph-mon[12201]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:38 smithi070 ceph-mon[17138]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:39.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:40.324 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:40 smithi070 ceph-mon[12201]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:40.325 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:39] "GET /metrics HTTP/1.1" 200 208022 "" "Prometheus/2.33.4" 2024-08-31T02:22:40.325 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:40 smithi070 ceph-mon[17138]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:40.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:40 smithi102 ceph-mon[13158]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:40.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:22:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:41.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[12201]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[17138]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:42.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:41 smithi102 ceph-mon[13158]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:42.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:42.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:42.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:22:42.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:22:42.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:22:42.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:22:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:22:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:22:43.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:43 smithi070 ceph-mon[12201]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:43.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:43 smithi070 ceph-mon[17138]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:44.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:43 smithi102 ceph-mon[13158]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:46.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:46 smithi102 ceph-mon[13158]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:46 smithi070 ceph-mon[12201]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:46.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:46 smithi070 ceph-mon[17138]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:48 smithi102 ceph-mon[13158]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:48 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[12201]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:22:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:22:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[17138]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:48 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:22:48.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:49.214 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:49.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:50 smithi102 ceph-mon[13158]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:50.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:50 smithi070 ceph-mon[12201]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:50.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:50 smithi070 ceph-mon[17138]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:50.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:49] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-08-31T02:22:52.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:52 smithi102 ceph-mon[13158]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:52 smithi070 ceph-mon[12201]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:52.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:52 smithi070 ceph-mon[17138]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:54.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:54 smithi102 ceph-mon[13158]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:54 smithi070 ceph-mon[12201]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:54.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:54 smithi070 ceph-mon[17138]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:56.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:56 smithi102 ceph-mon[13158]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:56 smithi070 ceph-mon[12201]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:56.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:56 smithi070 ceph-mon[17138]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:22:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:58 smithi102 ceph-mon[13158]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:58.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:58 smithi070 ceph-mon[12201]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:58.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:58 smithi070 ceph-mon[17138]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:22:58.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:22:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:22:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:22:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:22:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:22:59.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:22:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:00.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:00 smithi102 ceph-mon[13158]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:00.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:00 smithi070 ceph-mon[12201]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:22:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:22:59] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-08-31T02:23:00.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:00 smithi070 ceph-mon[17138]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:01.169 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:23:00] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:02.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:01 smithi102 ceph-mon[13158]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:02.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:01 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:02.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:01 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[12201]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[17138]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:01 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:03.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:03 smithi070 ceph-mon[12201]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:03.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:03 smithi070 ceph-mon[17138]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:03 smithi102 ceph-mon[13158]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:23:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:23:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:23:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:05.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:23:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:06 smithi102 ceph-mon[13158]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:06.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:06 smithi070 ceph-mon[12201]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:06 smithi070 ceph-mon[17138]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:08 smithi102 ceph-mon[13158]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[12201]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:23:08] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[17138]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:08.799 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:09 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:09.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:09 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:09.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:09 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:10 smithi102 ceph-mon[13158]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:10 smithi070 ceph-mon[12201]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:10.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:09] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-08-31T02:23:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:10 smithi070 ceph-mon[17138]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:12 smithi102 ceph-mon[13158]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:12 smithi070 ceph-mon[12201]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:12.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:12 smithi070 ceph-mon[17138]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:14 smithi102 ceph-mon[13158]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:14.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:14 smithi070 ceph-mon[12201]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:14 smithi070 ceph-mon[17138]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:16 smithi102 ceph-mon[13158]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:16 smithi070 ceph-mon[12201]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:16 smithi070 ceph-mon[17138]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:18 smithi102 ceph-mon[13158]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:18 smithi070 ceph-mon[12201]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:18 smithi070 ceph-mon[17138]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:20 smithi102 ceph-mon[13158]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:20 smithi070 ceph-mon[12201]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:20.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:19] "GET /metrics HTTP/1.1" 200 208031 "" "Prometheus/2.33.4" 2024-08-31T02:23:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:20 smithi070 ceph-mon[17138]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:21.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:23:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:22.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[12201]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:22.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:22.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:22.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[17138]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:22.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:22.160 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:21 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:22.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:21 smithi102 ceph-mon[13158]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:21 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:23 smithi102 ceph-mon[13158]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:24.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:23 smithi070 ceph-mon[12201]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:24.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:23 smithi070 ceph-mon[17138]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:26 smithi102 ceph-mon[13158]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:26 smithi070 ceph-mon[12201]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:26 smithi070 ceph-mon[17138]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:28 smithi102 ceph-mon[13158]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:28 smithi070 ceph-mon[12201]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:28.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:23:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:28.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:28 smithi070 ceph-mon[17138]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:28.831 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:29.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:29.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:29.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:29.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:29.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:30 smithi070 ceph-mon[12201]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:30.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:29] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-08-31T02:23:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:30 smithi070 ceph-mon[17138]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:30.645 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:30 smithi102 ceph-mon[13158]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:32 smithi070 ceph-mon[12201]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:32 smithi070 ceph-mon[17138]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:32 smithi102 ceph-mon[13158]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:34 smithi070 ceph-mon[12201]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:34 smithi070 ceph-mon[17138]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:34 smithi102 ceph-mon[13158]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:36 smithi070 ceph-mon[12201]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:36 smithi070 ceph-mon[17138]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:36 smithi102 ceph-mon[13158]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:38 smithi070 ceph-mon[12201]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:38 smithi070 ceph-mon[17138]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:38.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:38 smithi102 ceph-mon[13158]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:40.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:40 smithi070 ceph-mon[12201]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:40.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:39] "GET /metrics HTTP/1.1" 200 208008 "" "Prometheus/2.33.4" 2024-08-31T02:23:40.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:40 smithi070 ceph-mon[17138]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:40 smithi102 ceph-mon[13158]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:41.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:23:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[12201]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[17138]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:42.250 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:41 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:41 smithi102 ceph-mon[13158]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:41 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:23:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:23:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:23:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:23:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:23:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:23:44.348 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:43 smithi070 ceph-mon[12201]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:44.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:43 smithi070 ceph-mon[17138]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:43 smithi102 ceph-mon[13158]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:46.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:46 smithi102 ceph-mon[13158]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:46 smithi070 ceph-mon[12201]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:46.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:46 smithi070 ceph-mon[17138]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:48 smithi102 ceph-mon[13158]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:48 smithi070 ceph-mon[12201]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:48.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:48 smithi070 ceph-mon[17138]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:23:48] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:23:48.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:49.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:49.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:49 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:49.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:49 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:23:50.273 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:49] "GET /metrics HTTP/1.1" 200 208016 "" "Prometheus/2.33.4" 2024-08-31T02:23:50.632 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:50 smithi102 ceph-mon[13158]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:50 smithi070 ceph-mon[12201]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:50 smithi070 ceph-mon[17138]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:52 smithi102 ceph-mon[13158]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:52 smithi070 ceph-mon[12201]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:52 smithi070 ceph-mon[17138]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:54.658 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:54 smithi070 ceph-mon[12201]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:54.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:54 smithi070 ceph-mon[17138]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:54 smithi102 ceph-mon[13158]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:55.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-31T02:23:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:56 smithi102 ceph-mon[13158]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:56 smithi070 ceph-mon[17138]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:56 smithi070 ceph-mon[12201]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:23:58.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-31T02:23:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:58 smithi102 ceph-mon[13158]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:58.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:58 smithi070 ceph-mon[12201]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:58.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:58 smithi070 ceph-mon[17138]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:23:58.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:23:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:23:59.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:23:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:59.546 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:23:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:23:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:23:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:00.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:23:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:23:59] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-08-31T02:24:00.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:00 smithi070 ceph-mon[12201]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:00.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:00 smithi070 ceph-mon[17138]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:00 smithi102 ceph-mon[13158]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:00.923 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-31T02:24:01.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:24:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:02.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[17138]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[12201]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:02 smithi102 ceph-mon[13158]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[12201]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[17138]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:04.359 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:04.360 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:04.360 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:04 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:04 smithi102 ceph-mon[13158]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:04 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:05.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:24:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:06 smithi102 ceph-mon[13158]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:06.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:06 smithi070 ceph-mon[12201]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:06.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:06 smithi070 ceph-mon[17138]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:07.857 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 6s ago 9m - - 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 19s ago 9m - - 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 6s ago 4m 23.9M - ba2b418f427c 69d06c9f9ec4 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (4m) 19s ago 4m 55.0M - 8.3.5 dad864ee21e9 5683b141a5de 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (3m) 6s ago 3m 48.9M - 3.5 e1d6a67b021e 440a946c4497 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443 running (8m) 19s ago 8m 411M - 17.2.0 e1d6a67b021e 1c81198983cc 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283 running (10m) 6s ago 10m 462M - 17.2.0 e1d6a67b021e 4a3942799177 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (10m) 6s ago 10m 49.9M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (9m) 19s ago 9m 43.7M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (9m) 6s ago 9m 42.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 6s ago 5m 18.7M - 1dbe0e931976 31e55bbd1f8e 2024-08-31T02:24:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (4m) 19s ago 4m 16.4M - 1dbe0e931976 624dce24fc3e 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (8m) 6s ago 8m 52.8M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (8m) 6s ago 8m 53.1M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (7m) 6s ago 7m 46.3M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (7m) 6s ago 7m 48.5M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (7m) 19s ago 7m 49.5M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (6m) 19s ago 6m 46.9M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (6m) 19s ago 6m 48.5M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (5m) 19s ago 5m 51.5M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (3m) 19s ago 5m 66.3M - 514e6a882f6e 4d84e2075a19 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (3m) 6s ago 3m 72.6M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:24:07.859 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (3m) 19s ago 3m 71.0M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:24:08.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[12201]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:08.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[17138]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:08.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:07 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:07 smithi102 ceph-mon[13158]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:07 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:07 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:08.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:24:08] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:08.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions' 2024-08-31T02:24:08.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[12201]: from='client.24865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[17138]: from='client.24865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:09.082 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:09.083 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:08 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:08 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:08 smithi102 ceph-mon[13158]: from='client.24865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:08 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:10.167 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:09 smithi070 ceph-mon[12201]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:10.167 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:09] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-08-31T02:24:10.167 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:09 smithi070 ceph-mon[17138]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:10.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:09 smithi102 ceph-mon[13158]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:10.647 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:24:10.648 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:24:10.649 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:24:11.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:11 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1443940512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:24:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:11 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1443940512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:24:11.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:11 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1443940512' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:24:11.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph -s' 2024-08-31T02:24:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:12 smithi102 ceph-mon[13158]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:12 smithi070 ceph-mon[12201]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:12.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:12 smithi070 ceph-mon[17138]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:13.584 INFO:teuthology.orchestra.run.smithi070.stdout: cluster: 2024-08-31T02:24:13.584 INFO:teuthology.orchestra.run.smithi070.stdout: id: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:24:13.584 INFO:teuthology.orchestra.run.smithi070.stdout: health: HEALTH_OK 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: services: 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: mon: 3 daemons, quorum a,c,b (age 9m) 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: mgr: y(active, since 5m), standbys: x 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 6m) 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: data: 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: pools: 6 pools, 161 pgs 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: objects: 241 objects, 585 KiB 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: pgs: 161 active+clean 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:24:13.585 INFO:teuthology.orchestra.run.smithi070.stdout: io: 2024-08-31T02:24:13.586 INFO:teuthology.orchestra.run.smithi070.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-08-31T02:24:13.586 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:24:14.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:14 smithi070 ceph-mon[12201]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:14.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:14 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1669279573' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:24:14.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:14 smithi070 ceph-mon[17138]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:14.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:14 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1669279573' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:24:14.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ls' 2024-08-31T02:24:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:14 smithi102 ceph-mon[13158]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:14 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1669279573' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:24:16.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:16 smithi070 ceph-mon[12201]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:16.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:16 smithi070 ceph-mon[17138]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:agent 2/2 15s ago 5m * 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager ?:9093,9094 1/1 15s ago 5m smithi070=a;count:1 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:grafana ?:3000 1/1 8s ago 5m smithi102=a;count:1 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo 1/1 15s ago 3m count:1 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:mgr 2/2 15s ago 9m smithi070=y;smithi102=x;count:2 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:mon 3/3 15s ago 9m smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi102:172.21.15.102=b;count:3 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter ?:9100 2/2 15s ago 5m smithi070=a;smithi102=b;count:2 2024-08-31T02:24:16.398 INFO:teuthology.orchestra.run.smithi070.stdout:osd 8 15s ago - 2024-08-31T02:24:16.399 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus ?:9095 1/1 8s ago 5m smithi102=a;count:1 2024-08-31T02:24:16.399 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo ?:8000 2/2 15s ago 3m count:2 2024-08-31T02:24:16.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:16 smithi102 ceph-mon[13158]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:17.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-08-31T02:24:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:18 smithi102 ceph-mon[13158]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:18 smithi102 ceph-mon[13158]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:18 smithi070 ceph-mon[12201]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:18 smithi070 ceph-mon[12201]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:18 smithi070 ceph-mon[17138]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:18 smithi070 ceph-mon[17138]: from='client.24883 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:19.787 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled to redeploy mgr.x on host 'smithi102' 2024-08-31T02:24:20.127 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:19] "GET /metrics HTTP/1.1" 200 207972 "" "Prometheus/2.33.4" 2024-08-31T02:24:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:20 smithi102 ceph-mon[13158]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:20 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1693784645' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:20 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:20 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[12201]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1693784645' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:20.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[17138]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1693784645' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:20 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:24:20.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps --refresh' 2024-08-31T02:24:21.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:24:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:21 smithi070 ceph-mon[12201]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:21 smithi070 ceph-mon[12201]: Schedule redeploy daemon mgr.x 2024-08-31T02:24:21.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:21 smithi070 ceph-mon[17138]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:21.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:21 smithi070 ceph-mon[17138]: Schedule redeploy daemon mgr.x 2024-08-31T02:24:21.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:21 smithi102 ceph-mon[13158]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:21.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:21 smithi102 ceph-mon[13158]: Schedule redeploy daemon mgr.x 2024-08-31T02:24:22.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[17138]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:22.344 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.344 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[12201]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:22.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:22 smithi102 ceph-mon[13158]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:22.948 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:24:22.948 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 1s ago 10m - - 2024-08-31T02:24:22.948 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 14s ago 9m - - 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 1s ago 4m 22.4M - ba2b418f427c 69d06c9f9ec4 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (4m) 14s ago 4m 54.6M - 8.3.5 dad864ee21e9 5683b141a5de 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (3m) 1s ago 3m 49.0M - 3.5 e1d6a67b021e 440a946c4497 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443 running (9m) 14s ago 9m 411M - 17.2.0 e1d6a67b021e 1c81198983cc 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283 running (10m) 1s ago 10m 463M - 17.2.0 e1d6a67b021e 4a3942799177 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (10m) 1s ago 10m 50.6M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (9m) 14s ago 9m 43.7M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (9m) 1s ago 9m 42.8M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 1s ago 5m 18.6M - 1dbe0e931976 31e55bbd1f8e 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (5m) 14s ago 5m 16.5M - 1dbe0e931976 624dce24fc3e 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (8m) 1s ago 8m 53.0M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (8m) 1s ago 8m 53.3M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:24:22.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (8m) 1s ago 8m 46.4M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (7m) 1s ago 7m 48.7M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (7m) 14s ago 7m 49.7M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (6m) 14s ago 6m 47.1M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (6m) 14s ago 6m 48.7M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (6m) 14s ago 6m 51.7M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (4m) 14s ago 5m 66.3M - 514e6a882f6e 4d84e2075a19 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (3m) 1s ago 3m 72.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:24:22.950 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (3m) 14s ago 3m 71.1M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:24:23.768 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:24:24.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:24 smithi070 ceph-mon[17138]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:24.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:24 smithi070 ceph-mon[17138]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:24.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:24 smithi070 ceph-mon[12201]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:24.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:24 smithi070 ceph-mon[12201]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:24 smithi102 ceph-mon[13158]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:24 smithi102 ceph-mon[13158]: from='client.24901 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:24:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:26 smithi070 ceph-mon[12201]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:26.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:26 smithi070 ceph-mon[17138]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:26.469 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:26 smithi102 ceph-mon[13158]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:28 smithi070 ceph-mon[12201]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:28.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:28 smithi070 ceph-mon[17138]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:28.466 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:28 smithi102 ceph-mon[13158]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:28.821 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:24:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:28.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:30 smithi070 ceph-mon[12201]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:30 smithi070 ceph-mon[17138]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:30.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:29] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-08-31T02:24:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:30 smithi102 ceph-mon[13158]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:32 smithi070 ceph-mon[12201]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:32 smithi070 ceph-mon[17138]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:32.467 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:32 smithi102 ceph-mon[13158]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:34 smithi070 ceph-mon[12201]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:34 smithi070 ceph-mon[17138]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:34 smithi102 ceph-mon[13158]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:36 smithi070 ceph-mon[12201]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:36 smithi070 ceph-mon[17138]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:36 smithi102 ceph-mon[13158]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:38 smithi070 ceph-mon[12201]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:38 smithi070 ceph-mon[17138]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:38.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:38 smithi102 ceph-mon[13158]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:40.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:40 smithi070 ceph-mon[12201]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:40.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:39] "GET /metrics HTTP/1.1" 200 207975 "" "Prometheus/2.33.4" 2024-08-31T02:24:40.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:40 smithi070 ceph-mon[17138]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:40 smithi102 ceph-mon[13158]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:41.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:24:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:42 smithi102 ceph-mon[13158]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[12201]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[17138]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:24:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:24:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:24:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:24:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:24:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:24:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:44 smithi102 ceph-mon[13158]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:44 smithi070 ceph-mon[12201]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:44 smithi070 ceph-mon[17138]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:46.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:46 smithi102 ceph-mon[13158]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:46 smithi070 ceph-mon[12201]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:46 smithi070 ceph-mon[17138]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:48.570 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:48 smithi070 ceph-mon[12201]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:48.570 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:48 smithi070 ceph-mon[17138]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:48.571 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:48 smithi102 ceph-mon[13158]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:48.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:48.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:24:48] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:49.599 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:49 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:49 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:50.187 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:49] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-08-31T02:24:50.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:50 smithi070 ceph-mon[12201]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:50.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:50 smithi070 ceph-mon[17138]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:50 smithi102 ceph-mon[13158]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:52 smithi070 ceph-mon[12201]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:52.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:52 smithi070 ceph-mon[17138]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:52.576 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:52 smithi102 ceph-mon[13158]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:24:53.976 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:24:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:24:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:24:54.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:24:55.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:54 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:24:55.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:54 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:24:55.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:54 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:24:56.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:56 smithi102 ceph-mon[13158]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:56 smithi070 ceph-mon[12201]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:56.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:56 smithi070 ceph-mon[17138]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:24:58.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:58 smithi102 ceph-mon[13158]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:58.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:58 smithi070 ceph-mon[12201]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:58.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:58 smithi070 ceph-mon[17138]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:24:58.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:24:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:24:59.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:24:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:24:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:24:59.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:24:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:00.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:00 smithi102 ceph-mon[13158]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:00.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:00 smithi070 ceph-mon[12201]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:00.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:00 smithi070 ceph-mon[17138]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:24:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:24:59] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-08-31T02:25:01.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:25:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:02 smithi102 ceph-mon[13158]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[12201]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[17138]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:04 smithi102 ceph-mon[13158]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:04.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:04 smithi070 ceph-mon[12201]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:04.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:04 smithi070 ceph-mon[17138]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:06.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:06 smithi102 ceph-mon[13158]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:06.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:06 smithi070 ceph-mon[12201]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:06 smithi070 ceph-mon[17138]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:08.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:08 smithi102 ceph-mon[13158]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:08 smithi070 ceph-mon[12201]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:08 smithi070 ceph-mon[17138]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:08.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:25:08.925 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:08 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:25:08] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:09.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:09.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:09.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:09.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:09 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:09.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:09 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:10.032 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:09] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-08-31T02:25:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:10 smithi102 ceph-mon[13158]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:10 smithi070 ceph-mon[12201]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:10.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:10 smithi070 ceph-mon[17138]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:12.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:12 smithi102 ceph-mon[13158]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:12.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:12 smithi070 ceph-mon[12201]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:12.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:12 smithi070 ceph-mon[17138]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:14 smithi102 ceph-mon[13158]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:14.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:14 smithi070 ceph-mon[12201]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:14.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:14 smithi070 ceph-mon[17138]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:16.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:16 smithi102 ceph-mon[13158]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:16 smithi070 ceph-mon[12201]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:16 smithi070 ceph-mon[17138]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:18.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:18 smithi102 ceph-mon[13158]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:18 smithi070 ceph-mon[12201]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:18.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:18 smithi070 ceph-mon[17138]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[14678]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:25:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:20.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:20 smithi102 ceph-mon[13158]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:20 smithi070 ceph-mon[12201]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:20.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:19] "GET /metrics HTTP/1.1" 200 207976 "" "Prometheus/2.33.4" 2024-08-31T02:25:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:20 smithi070 ceph-mon[17138]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:21.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:25:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:22.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:22 smithi102 ceph-mon[13158]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[12201]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[17138]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:22.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:23.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:25:23.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:23.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:24 smithi102 ceph-mon[13158]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:24.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:24 smithi070 ceph-mon[12201]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:24.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:24 smithi070 ceph-mon[17138]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:26.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:26 smithi102 ceph-mon[13158]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:26 smithi070 ceph-mon[12201]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:26 smithi070 ceph-mon[17138]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:27.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:25:27.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 bash[37670]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr.x" found: no such container 2024-08-31T02:25:27.480 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 podman[37681]: 2024-08-31 02:25:27.177161329 +0000 UTC m=+0.138433755 container died 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, ceph=True, name=centos-stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8) 2024-08-31T02:25:27.815 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 podman[37681]: 2024-08-31 02:25:27.571950564 +0000 UTC m=+0.533222984 container cleanup 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, release=754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vcs-type=git, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-08-31T02:25:27.815 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 bash[37681]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:25:27.815 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 bash[37734]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr.x" found: no such container 2024-08-31T02:25:28.165 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 podman[37707]: 2024-08-31 02:25:27.815195879 +0000 UTC m=+0.640403930 container remove 1c81198983ccd74fd603b980f17f73f7d3a8e1f378d2b7ab94b91892486552ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, version=8, ceph=True, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-08-31T02:25:28.165 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:25:28.166 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-31T02:25:28.166 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 systemd[1]: Stopped Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:25:28.166 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:27 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 17.613s CPU time. 2024-08-31T02:25:28.166 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:25:28.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:28 smithi070 ceph-mon[12201]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:28 smithi070 ceph-mon[17138]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-mon[13158]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:28.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 podman[37863]: 2024-08-31 02:25:28.176446822 +0000 UTC m=+0.017586106 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:25:28.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 podman[37863]: 2024-08-31 02:25:28.299631841 +0000 UTC m=+0.140771108 container create 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-31T02:25:28.659 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:28.352Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:25:28.680 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 podman[37863]: 2024-08-31 02:25:28.544354403 +0000 UTC m=+0.385493676 container init 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-31T02:25:28.680 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 podman[37863]: 2024-08-31 02:25:28.547031972 +0000 UTC m=+0.388171242 container start 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-34b9d27) 2024-08-31T02:25:28.680 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-mgr[37911]: -- 172.21.15.102:0/3862071413 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5610b528b0e0 con 0x5610b5269000 2024-08-31T02:25:28.681 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 bash[37863]: 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d 2024-08-31T02:25:28.681 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:25:28.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:25:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:28.981 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:28.679+0000 7fc6657d9040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:25:28.981 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:28.762+0000 7fc6657d9040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:25:28.981 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:28.916+0000 7fc6657d9040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:25:29.295 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:28.985+0000 7fc6657d9040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:25:29.295 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:29.139+0000 7fc6657d9040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:25:29.295 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:29.295 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.296 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:25:29.422 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:25:29.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:25:29.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:29.576 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:29.294+0000 7fc6657d9040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:25:29.829 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:29.635+0000 7fc6657d9040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:25:29.829 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:29.702+0000 7fc6657d9040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:25:30.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:29 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:29.899+0000 7fc6657d9040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:25:30.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:30 smithi070 ceph-mon[12201]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:30.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:29] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-08-31T02:25:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:30 smithi070 ceph-mon[17138]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:30.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:30 smithi102 ceph-mon[13158]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:30.926 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:30 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:30.815+0000 7fc6657d9040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:25:30.926 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:30 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:30.884+0000 7fc6657d9040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:25:31.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:31 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:31.655+0000 7fc6657d9040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:25:31.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:31 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:31.772+0000 7fc6657d9040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:25:32.175 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:31 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:31.923+0000 7fc6657d9040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:25:32.175 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:31 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:31.986+0000 7fc6657d9040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:25:32.175 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.138+0000 7fc6657d9040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:25:32.444 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-mon[13158]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:32.445 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.219+0000 7fc6657d9040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:25:32.445 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.282+0000 7fc6657d9040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:25:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:32 smithi070 ceph-mon[12201]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:32 smithi070 ceph-mon[17138]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:32.725 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.444+0000 7fc6657d9040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:25:32.725 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.512+0000 7fc6657d9040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:25:32.725 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.580+0000 7fc6657d9040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:25:32.725 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.664+0000 7fc6657d9040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:25:32.725 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.724+0000 7fc6657d9040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:25:33.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:32.784+0000 7fc6657d9040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:25:33.759 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:25:33.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:25:33.759 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:33.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:25:33.760 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:33.605+0000 7fc6657d9040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:25:33] ENGINE Bus STARTING 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: CherryPy Checker: 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: The Application mounted at '' has an empty config. 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:25:33] ENGINE Serving on http://:::9283 2024-08-31T02:25:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:25:33] ENGINE Bus STARTED 2024-08-31T02:25:34.177 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:25:33] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:34.177 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:33.877Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:25:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:25:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:25:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:25:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:25:34.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:25:34.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:34.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:34 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:25:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:25:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:34 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:35 smithi070 ceph-mon[12201]: mgrmap e21: y(active, since 6m), standbys: x 2024-08-31T02:25:35.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:35 smithi070 ceph-mon[17138]: mgrmap e21: y(active, since 6m), standbys: x 2024-08-31T02:25:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:35 smithi102 ceph-mon[13158]: mgrmap e21: y(active, since 6m), standbys: x 2024-08-31T02:25:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:36 smithi070 ceph-mon[12201]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:36 smithi070 ceph-mon[17138]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:36 smithi102 ceph-mon[13158]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:38 smithi070 ceph-mon[12201]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:38 smithi070 ceph-mon[17138]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:38.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:38 smithi102 ceph-mon[13158]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:25:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:40.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:40 smithi070 ceph-mon[12201]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:40.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:40 smithi070 ceph-mon[17138]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:40.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:39] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-08-31T02:25:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:40 smithi102 ceph-mon[13158]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:41.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:25:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:42 smithi102 ceph-mon[13158]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[12201]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[17138]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:43.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:25:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:25:43.508 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:25:43.508 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:25:43.509 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:25:43.509 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:25:43.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:25:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:43.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:43.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:43.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:25:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:44 smithi102 ceph-mon[13158]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:44 smithi070 ceph-mon[12201]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:44 smithi070 ceph-mon[17138]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:46.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:46 smithi070 ceph-mon[12201]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:46.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:46 smithi070 ceph-mon[17138]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:46 smithi102 ceph-mon[13158]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:48.330 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:48 smithi070 ceph-mon[12201]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:48.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:48 smithi070 ceph-mon[17138]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:48.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:48 smithi102 ceph-mon[13158]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:48.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:25:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:49.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:50.220 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:49] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-08-31T02:25:50.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:50 smithi102 ceph-mon[13158]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:50 smithi070 ceph-mon[12201]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:50 smithi070 ceph-mon[17138]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:52.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:25:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[12201]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[17138]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:52 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:52.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:52 smithi102 ceph-mon[13158]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:52 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:25:53.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:25:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:53.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:25:53.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:25:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:25:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:25:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:54 smithi102 ceph-mon[13158]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:54 smithi070 ceph-mon[12201]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:54 smithi070 ceph-mon[17138]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:56.487 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:56 smithi102 ceph-mon[13158]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:56 smithi070 ceph-mon[12201]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:56 smithi070 ceph-mon[17138]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:25:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:58 smithi102 ceph-mon[13158]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:58.670 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:58 smithi070 ceph-mon[12201]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:58.671 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:58 smithi070 ceph-mon[17138]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:25:58.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:25:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:25:59.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:25:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:59.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:25:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:25:59.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:25:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:00.237 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:25:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:25:59] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-08-31T02:26:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:00 smithi102 ceph-mon[13158]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:00 smithi070 ceph-mon[12201]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:00 smithi070 ceph-mon[17138]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:01.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:26:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:02.508 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:02 smithi102 ceph-mon[13158]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:02.508 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:02.508 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[12201]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[17138]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:03.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:03.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:03.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:04 smithi102 ceph-mon[13158]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:04 smithi070 ceph-mon[12201]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:04 smithi070 ceph-mon[17138]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:06.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:06 smithi070 ceph-mon[12201]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:06.501 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:06 smithi070 ceph-mon[17138]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:06 smithi102 ceph-mon[13158]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:08.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:08 smithi102 ceph-mon[13158]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:08.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:08 smithi070 ceph-mon[12201]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:08.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:08 smithi070 ceph-mon[17138]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:08.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:09 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:09.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:09 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:09.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:09 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:10.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:09] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-08-31T02:26:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:10 smithi102 ceph-mon[13158]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:10 smithi070 ceph-mon[12201]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:10 smithi070 ceph-mon[17138]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:12 smithi102 ceph-mon[13158]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:12 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:12.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:12 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:12.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:26:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[12201]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[17138]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:12.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:12 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:13.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:13.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:13.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:13.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:13.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:14 smithi102 ceph-mon[13158]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:14 smithi070 ceph-mon[12201]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:14 smithi070 ceph-mon[17138]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:16.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:16 smithi102 ceph-mon[13158]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:16 smithi070 ceph-mon[12201]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:16 smithi070 ceph-mon[17138]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:18.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:18 smithi102 ceph-mon[13158]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:18 smithi070 ceph-mon[12201]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:18 smithi070 ceph-mon[17138]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:19.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:19.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:19.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:20.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:19] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-08-31T02:26:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:20 smithi102 ceph-mon[13158]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:20 smithi070 ceph-mon[12201]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:20 smithi070 ceph-mon[17138]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:22.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:26:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:22 smithi102 ceph-mon[13158]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[12201]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[17138]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:23.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:23.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:24 smithi102 ceph-mon[13158]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:24 smithi070 ceph-mon[12201]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:24 smithi070 ceph-mon[17138]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:26 smithi102 ceph-mon[13158]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:26 smithi070 ceph-mon[12201]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:26 smithi070 ceph-mon[17138]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:28.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:28 smithi102 ceph-mon[13158]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:28.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:28 smithi070 ceph-mon[12201]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:28.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:28 smithi070 ceph-mon[17138]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:28.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:26:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:26:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:26:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:26:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:30.074 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:29] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-08-31T02:26:30.382 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:30 smithi070 ceph-mon[12201]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:30.383 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:30 smithi070 ceph-mon[17138]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:30 smithi102 ceph-mon[13158]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[12201]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:32.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:26:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[17138]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:32 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:32 smithi102 ceph-mon[13158]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:32 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:32 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:33.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:33.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:33.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:33.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:34 smithi102 ceph-mon[13158]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:34 smithi070 ceph-mon[12201]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:34 smithi070 ceph-mon[17138]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:36 smithi102 ceph-mon[13158]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:36 smithi070 ceph-mon[12201]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:36 smithi070 ceph-mon[17138]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:38.578 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:38 smithi102 ceph-mon[13158]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:38 smithi070 ceph-mon[12201]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:38 smithi070 ceph-mon[17138]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:38.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:39.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:39.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:40.309 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:39] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-08-31T02:26:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:40 smithi102 ceph-mon[13158]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:40 smithi070 ceph-mon[12201]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:40 smithi070 ceph-mon[17138]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:42.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:26:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:42 smithi102 ceph-mon[13158]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[12201]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[17138]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:26:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:26:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:26:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:26:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:26:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:26:43.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:43.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:43.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:44 smithi102 ceph-mon[13158]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:44 smithi070 ceph-mon[12201]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:44 smithi070 ceph-mon[17138]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:46 smithi102 ceph-mon[13158]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:46 smithi070 ceph-mon[12201]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:46 smithi070 ceph-mon[17138]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:48 smithi102 ceph-mon[13158]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:48.616 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:48 smithi070 ceph-mon[12201]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:48.616 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:48 smithi070 ceph-mon[17138]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:48.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:49.571 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:49.571 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:50.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:49] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-08-31T02:26:50.650 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:50 smithi070 ceph-mon[12201]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:50.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:50 smithi070 ceph-mon[17138]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:50 smithi102 ceph-mon[13158]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:51 smithi102 ceph-mon[13158]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:51 smithi070 ceph-mon[12201]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:51.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:51 smithi070 ceph-mon[17138]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:52.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:26:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:26:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:53 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:26:53.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:26:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:53.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:26:53.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:26:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:26:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:26:54.566 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:54 smithi102 ceph-mon[13158]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:54 smithi070 ceph-mon[12201]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:54 smithi070 ceph-mon[17138]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:56.512 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:56 smithi102 ceph-mon[13158]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:56 smithi070 ceph-mon[12201]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:56 smithi070 ceph-mon[17138]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:26:58.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:58 smithi102 ceph-mon[13158]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:58 smithi070 ceph-mon[12201]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:58 smithi070 ceph-mon[17138]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:26:58.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:26:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:26:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:26:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:59.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:26:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:26:59.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:26:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:00.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:26:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:26:59] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-08-31T02:27:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:00 smithi102 ceph-mon[13158]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:00 smithi070 ceph-mon[12201]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:00 smithi070 ceph-mon[17138]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:02.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:27:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:27:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:02 smithi102 ceph-mon[13158]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:02 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[12201]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:02.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[17138]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:02.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:02 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:03.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:03.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:03.950 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:04 smithi102 ceph-mon[13158]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:04 smithi070 ceph-mon[12201]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:04 smithi070 ceph-mon[17138]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:06 smithi102 ceph-mon[13158]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:06 smithi070 ceph-mon[12201]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:06 smithi070 ceph-mon[17138]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:08.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:08 smithi102 ceph-mon[13158]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:08.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:08 smithi070 ceph-mon[12201]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:08.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:08 smithi070 ceph-mon[17138]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:08.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:27:09.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:09 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:09.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:09 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:09 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:10.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:09] "GET /metrics HTTP/1.1" 200 207978 "" "Prometheus/2.33.4" 2024-08-31T02:27:10.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:10 smithi070 ceph-mon[12201]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:10.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:10 smithi070 ceph-mon[17138]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:10 smithi102 ceph-mon[13158]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:12 smithi102 ceph-mon[13158]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:12 smithi070 ceph-mon[12201]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:12.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:27:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:27:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:12 smithi070 ceph-mon[17138]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:13 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:13 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:13 smithi102 ceph-mon[13158]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[12201]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:13.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-mon[17138]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:13.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:13.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:13.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:16.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:16 smithi102 ceph-mon[13158]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:16 smithi070 ceph-mon[12201]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:16 smithi070 ceph-mon[17138]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:18 smithi102 ceph-mon[13158]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:18 smithi070 ceph-mon[12201]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:18 smithi070 ceph-mon[17138]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:18.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:27:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:19 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:19.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:19 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:19.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:19 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:20 smithi102 ceph-mon[13158]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:20.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:20 smithi070 ceph-mon[12201]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:20 smithi070 ceph-mon[17138]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:20.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:19] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-08-31T02:27:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:22 smithi102 ceph-mon[13158]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:22.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:22 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[12201]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:22.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:22.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:27:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:27:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[17138]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:22.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:22 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:23.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:23.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:24.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:24 smithi102 ceph-mon[13158]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:24.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:24 smithi070 ceph-mon[12201]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:24 smithi070 ceph-mon[17138]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:26 smithi102 ceph-mon[13158]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:26 smithi070 ceph-mon[12201]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:26 smithi070 ceph-mon[17138]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:28 smithi102 ceph-mon[13158]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:28 smithi070 ceph-mon[12201]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:28.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:28 smithi070 ceph-mon[17138]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:28.865 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:29.285 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:29.286 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:29 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:29 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:29.704 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:27:30.230 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:30 smithi070 ceph-mon[12201]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:30.230 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:29] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-08-31T02:27:30.230 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:30 smithi070 ceph-mon[17138]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:30 smithi102 ceph-mon[13158]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 10s ago 13m - - 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 19s ago 12m - - 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (7m) 10s ago 7m 25.3M - ba2b418f427c 69d06c9f9ec4 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (7m) 19s ago 7m 54.9M - 8.3.5 dad864ee21e9 5683b141a5de 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (6m) 10s ago 6m 54.8M - 3.5 e1d6a67b021e 440a946c4497 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (2m) 19s ago 12m 441M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:27:32.205 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283 running (14m) 10s ago 13m 470M - 17.2.0 e1d6a67b021e 4a3942799177 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (14m) 10s ago 14m 57.1M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (12m) 19s ago 12m 46.8M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (12m) 10s ago 12m 46.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (8m) 10s ago 8m 16.7M - 1dbe0e931976 31e55bbd1f8e 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (8m) 19s ago 8m 19.9M - 1dbe0e931976 624dce24fc3e 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (12m) 10s ago 12m 54.2M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (11m) 10s ago 11m 54.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (11m) 10s ago 11m 47.8M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (10m) 10s ago 10m 50.1M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (10m) 19s ago 10m 51.2M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (10m) 19s ago 10m 48.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (9m) 19s ago 9m 50.0M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (9m) 19s ago 9m 52.8M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (7m) 19s ago 8m 66.5M - 514e6a882f6e 4d84e2075a19 2024-08-31T02:27:32.206 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (6m) 10s ago 6m 73.8M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:27:32.207 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (6m) 19s ago 6m 72.1M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:27:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:32 smithi070 ceph-mon[12201]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:32 smithi070 ceph-mon[17138]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:32.500 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:32 smithi102 ceph-mon[13158]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:32.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.102 - - [31/Aug/2024:02:27:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:27:33.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[12201]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:27:33.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[17138]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:27:33.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions' 2024-08-31T02:27:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:33 smithi102 ceph-mon[13158]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:27:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:33 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:33 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:33.668 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:33.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:33.668 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:33.668 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:34 smithi070 ceph-mon[12201]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:34 smithi070 ceph-mon[17138]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:34 smithi102 ceph-mon[13158]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:27:36.102 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 1 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-08-31T02:27:36.103 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 1 2024-08-31T02:27:36.104 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:27:36.104 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:27:36.301 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:36 smithi102 ceph-mon[13158]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:36.301 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:36 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3687332812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:36 smithi070 ceph-mon[12201]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:36 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3687332812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:36 smithi070 ceph-mon[17138]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:36 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3687332812' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:36.768 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph -s' 2024-08-31T02:27:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:38 smithi102 ceph-mon[13158]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:38.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:38 smithi070 ceph-mon[12201]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:38 smithi070 ceph-mon[17138]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:38.526 INFO:teuthology.orchestra.run.smithi070.stdout: cluster: 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: id: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: health: HEALTH_OK 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: services: 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: mgr: y(active, since 8m), standbys: x 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: osd: 8 osds: 8 up (since 9m), 8 in (since 9m) 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: data: 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: pools: 6 pools, 161 pgs 2024-08-31T02:27:38.527 INFO:teuthology.orchestra.run.smithi070.stdout: objects: 241 objects, 585 KiB 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: usage: 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: pgs: 161 active+clean 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: io: 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-08-31T02:27:38.528 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:27:38.729 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:27:39.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:27:39.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:39 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:39.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:39 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1874354058' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:27:39.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:39 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:39.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:39 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1874354058' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:27:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:39 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:39 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1874354058' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:27:39.971 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:39] "GET /metrics HTTP/1.1" 200 207979 "" "Prometheus/2.33.4" 2024-08-31T02:27:40.372 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:40 smithi070 ceph-mon[12201]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:40.372 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:40 smithi070 ceph-mon[17138]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:40 smithi102 ceph-mon[13158]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:41.173 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_OK 2024-08-31T02:27:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:41 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1994998799' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:27:41.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:41 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1994998799' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:27:41.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:41 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1994998799' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:27:41.770 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-08-31T02:27:42.321 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 172.21.15.70 - - [31/Aug/2024:02:27:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-08-31T02:27:42.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[12201]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:42.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:42.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:42.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[17138]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:42.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:42.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:42 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:42 smithi102 ceph-mon[13158]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:42 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' 2024-08-31T02:27:43.415 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:43.415 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-mon[12201]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:43.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:43.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-mon[17138]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:43 smithi102 ceph-mon[13158]: from='mgr.14439 172.21.15.70:0/1055207609' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:43.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:43.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:43.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:43.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:43.898 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:27:44.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:44 smithi070 ceph-mon[12201]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:44.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:44 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3066692500' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:44.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:44 smithi070 ceph-mon[17138]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:44.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:44 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3066692500' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:44.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph mgr fail' 2024-08-31T02:27:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:44 smithi102 ceph-mon[13158]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:27:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:44 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3066692500' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:27:46.295 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:46 smithi070 ceph-mon[12201]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:46.295 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:46 smithi070 ceph-mon[17138]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:46 smithi102 ceph-mon[13158]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:27:47.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3226982358' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[12201]: osdmap e97: 8 total, 8 up, 8 in 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3226982358' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mon[17138]: osdmap e97: 8 total, 8 up, 8 in 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setuser ceph since I am not root 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ignoring --setgroup ceph since I am not root 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-mgr[12432]: -- 172.21.15.70:0/2761732071 <== mon.1 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fad40c6340 con 0x55fad409e800 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:47.393+0000 7fac49a17000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:27:47.476 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:47.474+0000 7fac49a17000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:27:47.591 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3226982358' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.591 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:27:47.591 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-mon[13158]: osdmap e97: 8 total, 8 up, 8 in 2024-08-31T02:27:47.591 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE Bus STOPPING 2024-08-31T02:27:47.751 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:47.659+0000 7fac49a17000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:27:47.751 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:47.693Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:27:47.832 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:27:47.852 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:27:47.852 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE Bus STOPPED 2024-08-31T02:27:47.852 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE Bus STARTING 2024-08-31T02:27:47.852 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE Serving on http://:::9283 2024-08-31T02:27:47.852 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:27:47] ENGINE Bus STARTED 2024-08-31T02:27:48.093 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:47.750+0000 7fac49a17000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:27:48.094 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:47.888+0000 7fac49a17000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:27:48.437 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:48.123+0000 7fac49a17000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:27:48.437 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:48.093Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:27:48.437 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: mgrmap e22: x(active, starting, since 0.59096s) 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:27:48.438 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: Manager daemon x is now available 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: mgrmap e22: x(active, starting, since 0.59096s) 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:27:48.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: Manager daemon x is now available 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:48.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.578 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:48] "GET /metrics HTTP/1.1" 200 34893 "" "Prometheus/2.33.4" 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: mgrmap e22: x(active, starting, since 0.59096s) 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:27:48.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: Manager daemon x is now available 2024-08-31T02:27:48.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:27:48.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:27:48.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:48.493+0000 7fac49a17000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:27:48.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:48.585+0000 7fac49a17000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:27:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:27:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.70:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:27:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:27:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:27:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.70:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:27:49.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:48.760+0000 7fac49a17000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Queued rgw.foo for migration 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Migrating certs/keys for iscsi.foo spec to cert store 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Migrating certs/keys for rgw.foo spec to cert store 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Checking for cert/key for grafana.a 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:27:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: mgrmap e23: x(active, since 1.5952s) 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Queued rgw.foo for migration 2024-08-31T02:27:49.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Migrating certs/keys for iscsi.foo spec to cert store 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Migrating certs/keys for rgw.foo spec to cert store 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Checking for cert/key for grafana.a 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: mgrmap e23: x(active, since 1.5952s) 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.490 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Queued rgw.foo for migration 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Migrating certs/keys for iscsi.foo spec to cert store 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Migrating certs/keys for rgw.foo spec to cert store 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Checking for cert/key for grafana.a 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: mgrmap e23: x(active, since 1.5952s) 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:49.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.493 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.494 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:49.494 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.494 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:27:49.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:27:49.495+0000 7fc5bc789640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:27:49] ENGINE Error in HTTPServer.serve 2024-08-31T02:27:49.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Traceback (most recent call last): 2024-08-31T02:27:49.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:27:49.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._connections.run(self.expiration_interval) 2024-08-31T02:27:49.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._run(expiration_interval) 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: s = self.context.wrap_socket( 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return self.sslsocket_class._create( 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self.do_handshake() 2024-08-31T02:27:49.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:27:49.920 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._sslobj.do_handshake() 2024-08-31T02:27:49.920 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:27:49.920 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:27:49.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:49.854+0000 7fac49a17000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:27:50.229 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:49.950+0000 7fac49a17000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:27:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: [31/Aug/2024:02:27:49] ENGINE Bus STARTING 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: [31/Aug/2024:02:27:49] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: [31/Aug/2024:02:27:49] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: [31/Aug/2024:02:27:49] ENGINE Bus STARTED 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:50 smithi102 ceph-mon[13158]: [31/Aug/2024:02:27:49] ENGINE Error in HTTPServer.serve 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:27:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:27:50.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:27:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: [31/Aug/2024:02:27:49] ENGINE Bus STARTING 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: [31/Aug/2024:02:27:49] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: [31/Aug/2024:02:27:49] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: [31/Aug/2024:02:27:49] ENGINE Bus STARTED 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[12201]: [31/Aug/2024:02:27:49] ENGINE Error in HTTPServer.serve 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:27:50.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: [31/Aug/2024:02:27:49] ENGINE Bus STARTING 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: [31/Aug/2024:02:27:49] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: [31/Aug/2024:02:27:49] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: [31/Aug/2024:02:27:49] ENGINE Bus STARTED 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-mon[17138]: [31/Aug/2024:02:27:49] ENGINE Error in HTTPServer.serve 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:27:50.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:27:50.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:27:51.123 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:50.861+0000 7fac49a17000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:27:51.123 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:51.084Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: Reconfiguring daemon agent.smithi070 on smithi070 2024-08-31T02:27:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: Deploying cephadm binary to smithi070 2024-08-31T02:27:51.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: mgrmap e24: x(active, since 3s) 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.215+0000 7fac49a17000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.369+0000 7fac49a17000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:27:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: Reconfiguring daemon agent.smithi070 on smithi070 2024-08-31T02:27:51.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: Deploying cephadm binary to smithi070 2024-08-31T02:27:51.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: mgrmap e24: x(active, since 3s) 2024-08-31T02:27:51.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: Reconfiguring daemon agent.smithi070 on smithi070 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: Deploying cephadm binary to smithi070 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: mgrmap e24: x(active, since 3s) 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:51 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:51.822 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:51 smithi070 systemd[1]: Stopping Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:27:51.822 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.462+0000 7fac49a17000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:27:51.823 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.579+0000 7fac49a17000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:27:51.823 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.672+0000 7fac49a17000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:27:51.823 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.821+0000 7fac49a17000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:27:52.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:51 smithi070 bash[51236]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter.a" found: no such container 2024-08-31T02:27:52.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:51.920+0000 7fac49a17000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:27:52.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:52.024+0000 7fac49a17000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:27:52.515 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:52.320+0000 7fac49a17000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:27:52.515 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:52.423+0000 7fac49a17000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:27:52.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[12201]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-31T02:27:52.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[12201]: Reconfiguring daemon node-exporter.a on smithi070 2024-08-31T02:27:52.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[12201]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:52.516 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[17138]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-31T02:27:52.516 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[17138]: Reconfiguring daemon node-exporter.a on smithi070 2024-08-31T02:27:52.516 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-mon[17138]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:52.516 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:52 smithi070 podman[51264]: 2024-08-31 02:27:52.351737761 +0000 UTC m=+0.451225951 container died 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:52.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:52 smithi102 ceph-mon[13158]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-31T02:27:52.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:52 smithi102 ceph-mon[13158]: Reconfiguring daemon node-exporter.a on smithi070 2024-08-31T02:27:52.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:52 smithi102 ceph-mon[13158]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:52.925 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:52.514+0000 7fac49a17000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:27:53.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-mon[12201]: mgrmap e25: x(active, since 5s) 2024-08-31T02:27:53.521 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-mon[17138]: mgrmap e25: x(active, since 5s) 2024-08-31T02:27:53.521 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 podman[51264]: 2024-08-31 02:27:53.460367905 +0000 UTC m=+1.559856092 container cleanup 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:53.521 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 bash[51264]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-31T02:27:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:53 smithi102 ceph-mon[13158]: mgrmap e25: x(active, since 5s) 2024-08-31T02:27:53.827 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=error ts=2024-08-31T02:27:53.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.70:8443//api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:27:53.827 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:53.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:27:53.827 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.102:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.102 because it doesn't contain any IP SANs" 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:53.606+0000 7fac49a17000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:27:53] ENGINE Bus STARTING 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: CherryPy Checker: 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: The Application mounted at '' has an empty config. 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:27:53] ENGINE Serving on http://:::9283 2024-08-31T02:27:53.828 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: [31/Aug/2024:02:27:53] ENGINE Bus STARTED 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 podman[51290]: 2024-08-31 02:27:53.678561568 +0000 UTC m=+1.658708279 container remove 31e55bbd1f8e0d339395f88e484af697bb2832b20cb1c3b66246e3e904b0a112 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 bash[51354]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter.a" found: no such container 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 systemd[1]: Stopped Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 3.937s CPU time. 2024-08-31T02:27:53.829 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 systemd[1]: Starting Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:27:54.097 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:27:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=warn ts=2024-08-31T02:27:53.827Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.70:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.70 because it doesn't contain any IP SANs" 2024-08-31T02:27:54.098 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 podman[51459]: 2024-08-31 02:27:53.844770683 +0000 UTC m=+0.016754191 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-08-31T02:27:54.098 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:53 smithi070 podman[51459]: 2024-08-31 02:27:53.971461338 +0000 UTC m=+0.143444844 container create a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: Standby manager daemon y started 2024-08-31T02:27:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:27:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: Standby manager daemon y started 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:27:54.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 podman[51459]: 2024-08-31 02:27:54.190541587 +0000 UTC m=+0.362525094 container init a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 podman[51459]: 2024-08-31 02:27:54.193273947 +0000 UTC m=+0.365257455 container start a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.196Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.196Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-08-31T02:27:54.451 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=arp 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=bcache 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=bonding 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=btrfs 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=conntrack 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=cpu 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=diskstats 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=dmi 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=edac 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=entropy 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-08-31T02:27:54.452 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=filefd 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=filesystem 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=hwmon 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=infiniband 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=ipvs 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=loadavg 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=mdadm 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=meminfo 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=netclass 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=netdev 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=netstat 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=nfs 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=nfsd 2024-08-31T02:27:54.453 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=nvme 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=os 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=pressure 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=rapl 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=schedstat 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=sockstat 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=softnet 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=stat 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=tapestats 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=textfile 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=time 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-08-31T02:27:54.454 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=uname 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=vmstat 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=xfs 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:115 level=info collector=zfs 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.197Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[51487]: ts=2024-08-31T02:27:54.198Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 bash[51459]: a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e 2024-08-31T02:27:54.455 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:27:54 smithi070 systemd[1]: Started Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: Standby manager daemon y started 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:27:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:54 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1601672488' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: mgrmap e26: x(active, since 7s), standbys: y 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[12201]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: mgrmap e26: x(active, since 7s), standbys: y 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:27:55.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:27:55.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:55 smithi070 ceph-mon[17138]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: mgrmap e26: x(active, since 7s), standbys: y 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-31T02:27:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:55 smithi102 ceph-mon[13158]: Reconfiguring daemon alertmanager.a on smithi070 2024-08-31T02:27:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:56 smithi070 ceph-mon[12201]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:27:56.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:56 smithi070 ceph-mon[17138]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:27:56.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:56 smithi102 ceph-mon[13158]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:27:58.299 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:58 smithi102 ceph-mon[13158]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:27:58.323 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:58 smithi070 ceph-mon[12201]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:27:58.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:58 smithi070 ceph-mon[17138]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:27:58.917 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:27:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:58] "GET /metrics HTTP/1.1" 200 34893 "" "Prometheus/2.33.4" 2024-08-31T02:27:59.346 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:27:59 smithi070 ceph-mon[12201]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:59.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:27:59 smithi070 ceph-mon[17138]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:27:59.408 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:27:59 smithi102 ceph-mon[13158]: from='client.14832 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:00.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:27:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:27:59] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:28:00.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:00 smithi070 ceph-mon[12201]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:28:00.546 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 systemd[1]: Stopping Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:00.547 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 bash[52067]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager.a" found: no such container 2024-08-31T02:28:00.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:00 smithi070 ceph-mon[17138]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:28:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:00 smithi102 ceph-mon[13158]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:28:00.855 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[37197]: level=info ts=2024-08-31T02:28:00.546Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:28:00.855 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 podman[52079]: 2024-08-31 02:28:00.566856147 +0000 UTC m=+0.115362137 container died 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:01.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 podman[52079]: 2024-08-31 02:28:00.962762281 +0000 UTC m=+0.511268262 container cleanup 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:01.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 bash[52079]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-31T02:28:01.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:00 smithi070 bash[52104]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager.a" found: no such container 2024-08-31T02:28:01.641 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 podman[52093]: 2024-08-31 02:28:01.339790449 +0000 UTC m=+0.772446322 container remove 69d06c9f9ec45f478fa2a548d68d7cc06aca7fd3c43e1b5151524db77f289529 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:01.641 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 podman[52093]: 2024-08-31 02:28:01.423616751 +0000 UTC m=+0.856272549 volume remove 3415c6c9a88c3a814c09d95f36535864b62ac780b8453a38279b664ce4f64713 2024-08-31T02:28:01.641 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-31T02:28:01.641 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 systemd[1]: Stopped Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:01.641 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 systemd[1]: Starting Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:01.925 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 podman[52167]: 2024-08-31 02:28:01.641080128 +0000 UTC m=+0.111296080 volume create 61cec06e566082613cb6422091ff20b052b2e901a2b2ad0b3f78c7f52b271077 2024-08-31T02:28:01.925 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 podman[52167]: 2024-08-31 02:28:01.545873882 +0000 UTC m=+0.016089848 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-08-31T02:28:01.925 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:01 smithi070 podman[52167]: 2024-08-31 02:28:01.741754917 +0000 UTC m=+0.211970911 container create ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 podman[52167]: 2024-08-31 02:28:02.052651589 +0000 UTC m=+0.522867541 container init ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 podman[52167]: 2024-08-31 02:28:02.055341948 +0000 UTC m=+0.525557918 container start ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.065Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.065Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.066Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.068Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.092Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.093Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.095Z caller=main.go:518 msg=Listening address=:9093 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:02.095Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 bash[52167]: ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 2024-08-31T02:28:02.451 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:02 smithi070 systemd[1]: Started Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:02.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.70:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.102:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:28:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:03 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:28:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:03 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:28:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:28:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:28:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:28:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:28:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:04.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:04 smithi070 ceph-mon[12201]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:04.325 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:04 smithi070 ceph-mon[17138]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:04.326 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:04.069Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001034448s 2024-08-31T02:28:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:04 smithi102 ceph-mon[13158]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:06.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:06 smithi070 ceph-mon[12201]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:06.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:06 smithi070 ceph-mon[17138]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:06 smithi102 ceph-mon[13158]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:28:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/719521229' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/224824235' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/719521229' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:28:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/224824235' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:07 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:07.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:07.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:07.850 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/719521229' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:28:07.850 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/224824235' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:07.850 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:07 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]: dispatch 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:28:08] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: Reconfiguring daemon agent.smithi102 on smithi102 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: Deploying cephadm binary to smithi102 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]': finished 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: osdmap e98: 8 total, 8 up, 8 in 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]: dispatch 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: Reconfiguring daemon agent.smithi102 on smithi102 2024-08-31T02:28:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: Deploying cephadm binary to smithi102 2024-08-31T02:28:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]': finished 2024-08-31T02:28:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: osdmap e98: 8 total, 8 up, 8 in 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]: dispatch 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: Reconfiguring daemon agent.smithi102 on smithi102 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: Deploying cephadm binary to smithi102 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1880793210"}]': finished 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: osdmap e98: 8 total, 8 up, 8 in 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]: dispatch 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:08.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]': finished 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: osdmap e99: 8 total, 8 up, 8 in 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2376735667' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:09 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:28:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:28:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]': finished 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: osdmap e99: 8 total, 8 up, 8 in 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2376735667' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/4255221815' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2316598650"}]': finished 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: osdmap e99: 8 total, 8 up, 8 in 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2376735667' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]: dispatch 2024-08-31T02:28:10.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:28:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[12427]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:28:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:28:10.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:10 smithi102 ceph-mon[13158]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-08-31T02:28:10.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:10 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]': finished 2024-08-31T02:28:10.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:10 smithi102 ceph-mon[13158]: osdmap e100: 8 total, 8 up, 8 in 2024-08-31T02:28:10.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:10 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/370330500' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:10.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:10 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[12201]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-08-31T02:28:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]': finished 2024-08-31T02:28:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[12201]: osdmap e100: 8 total, 8 up, 8 in 2024-08-31T02:28:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/370330500' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[17138]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1 op/s 2024-08-31T02:28:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2284893798"}]': finished 2024-08-31T02:28:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[17138]: osdmap e100: 8 total, 8 up, 8 in 2024-08-31T02:28:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/370330500' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:10 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]: dispatch 2024-08-31T02:28:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:11 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]': finished 2024-08-31T02:28:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:11 smithi102 ceph-mon[13158]: osdmap e101: 8 total, 8 up, 8 in 2024-08-31T02:28:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:11 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1866526832' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:11 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:11.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]': finished 2024-08-31T02:28:11.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[17138]: osdmap e101: 8 total, 8 up, 8 in 2024-08-31T02:28:11.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1866526832' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:11.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:11.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/3763771457"}]': finished 2024-08-31T02:28:11.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[12201]: osdmap e101: 8 total, 8 up, 8 in 2024-08-31T02:28:11.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1866526832' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:11.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:11 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]: dispatch 2024-08-31T02:28:12.333 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:28:12.072Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003636011s 2024-08-31T02:28:12.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.70:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.102:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:12.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:12.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 bash[41811]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:28:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[17138]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1 op/s 2024-08-31T02:28:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]': finished 2024-08-31T02:28:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[17138]: osdmap e102: 8 total, 8 up, 8 in 2024-08-31T02:28:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1417591735' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[12201]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1 op/s 2024-08-31T02:28:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]': finished 2024-08-31T02:28:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[12201]: osdmap e102: 8 total, 8 up, 8 in 2024-08-31T02:28:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1417591735' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:12 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.837 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-mon[13158]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1 op/s 2024-08-31T02:28:12.837 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/3763771457"}]': finished 2024-08-31T02:28:12.837 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-mon[13158]: osdmap e102: 8 total, 8 up, 8 in 2024-08-31T02:28:12.837 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1417591735' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.837 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]: dispatch 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.531Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:28:12.838 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[32326]: ts=2024-08-31T02:28:12.532Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:28:13.444 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 podman[41833]: 2024-08-31 02:28:13.171542524 +0000 UTC m=+0.767838981 container died 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:13.780 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:13 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]': finished 2024-08-31T02:28:13.780 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:13 smithi102 ceph-mon[13158]: osdmap e103: 8 total, 8 up, 8 in 2024-08-31T02:28:13.780 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:13 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]: dispatch 2024-08-31T02:28:13.781 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 podman[41833]: 2024-08-31 02:28:13.54385121 +0000 UTC m=+1.140147675 container cleanup 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:13.781 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 bash[41833]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:28:13.781 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 bash[41918]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:28:13.917 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]': finished 2024-08-31T02:28:13.918 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[12201]: osdmap e103: 8 total, 8 up, 8 in 2024-08-31T02:28:13.918 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]: dispatch 2024-08-31T02:28:13.918 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/663153968"}]': finished 2024-08-31T02:28:13.918 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[17138]: osdmap e103: 8 total, 8 up, 8 in 2024-08-31T02:28:13.918 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:13 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]: dispatch 2024-08-31T02:28:14.116 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 podman[41870]: 2024-08-31 02:28:13.78053842 +0000 UTC m=+1.238928752 container remove 4d84e2075a1937c0dbec4eb095c8f690ecc222920efe09eefed4b0f0aca66082 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:14.116 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:28:14.116 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:14.116 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.817s CPU time. 2024-08-31T02:28:14.116 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:13 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:14.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 podman[42009]: 2024-08-31 02:28:14.253614853 +0000 UTC m=+0.346181810 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:28:14.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 podman[42009]: 2024-08-31 02:28:14.350212478 +0000 UTC m=+0.442779438 container create c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[12201]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]': finished 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[12201]: osdmap e104: 8 total, 8 up, 8 in 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[17138]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]': finished 2024-08-31T02:28:14.618 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:14 smithi070 ceph-mon[17138]: osdmap e104: 8 total, 8 up, 8 in 2024-08-31T02:28:14.795 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-mon[13158]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 81 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:28:14.795 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3907141724' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/3182173953"}]': finished 2024-08-31T02:28:14.795 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-mon[13158]: osdmap e104: 8 total, 8 up, 8 in 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 podman[42009]: 2024-08-31 02:28:14.586248902 +0000 UTC m=+0.678815859 container init c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 podman[42009]: 2024-08-31 02:28:14.58916925 +0000 UTC m=+0.681736208 container start c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.612Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:28:14.796 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.614Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.614Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.615Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.617Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.617Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.089µs 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.617Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:14.618Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 bash[42009]: c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 2024-08-31T02:28:14.797 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:14 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.074Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.074Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.074Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.413µs wal_replay_duration=457.696542ms total_replay_duration=457.741569ms 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.078Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.078Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:28:15.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.078Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:28:15.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.098Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.852919ms db_storage=856ns remote_storage=1.329µs web_handler=297ns query_engine=632ns scrape=341.313µs scrape_sd=95.831µs notify=10.218µs notify_sd=8.361µs rules=18.916866ms 2024-08-31T02:28:15.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:15.098Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:28:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[12201]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-31T02:28:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[12201]: Reconfiguring daemon node-exporter.b on smithi102 2024-08-31T02:28:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[12201]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:28:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[17138]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-31T02:28:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[17138]: Reconfiguring daemon node-exporter.b on smithi102 2024-08-31T02:28:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:15 smithi070 ceph-mon[17138]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:28:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-mon[13158]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-31T02:28:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-mon[13158]: Reconfiguring daemon node-exporter.b on smithi102 2024-08-31T02:28:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:15 smithi102 ceph-mon[13158]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:28:16.918 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:16 smithi102 systemd[1]: Stopping Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:16.918 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:16 smithi102 bash[42389]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter.b" found: no such container 2024-08-31T02:28:17.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:17 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:17.278 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 podman[42399]: 2024-08-31 02:28:17.022356234 +0000 UTC m=+0.439742124 container died 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:17.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:17 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:17.424 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:17 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:17.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 podman[42399]: 2024-08-31 02:28:17.377755513 +0000 UTC m=+0.795141391 container cleanup 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:17.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 bash[42399]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-31T02:28:17.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 bash[42467]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter.b" found: no such container 2024-08-31T02:28:18.047 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 podman[42443]: 2024-08-31 02:28:17.716872476 +0000 UTC m=+0.696604188 container remove 624dce24fc3ebb514b473b48b89eb7a6c9d4dc760787a01ee8a15195be5377cd (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:18.047 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:28:18.047 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-31T02:28:18.048 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 systemd[1]: Stopped Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:18.048 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 3.882s CPU time. 2024-08-31T02:28:18.048 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:17 smithi102 systemd[1]: Starting Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:18.373 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-mon[13158]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:18.373 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:18.373 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 podman[42575]: 2024-08-31 02:28:18.178554512 +0000 UTC m=+0.344638713 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-08-31T02:28:18.373 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 podman[42575]: 2024-08-31 02:28:18.290795853 +0000 UTC m=+0.456880034 container create a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:18 smithi070 ceph-mon[12201]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:18.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:18 smithi070 ceph-mon[17138]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:18.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:18.918 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 podman[42575]: 2024-08-31 02:28:18.443071475 +0000 UTC m=+0.609155656 container init a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 podman[42575]: 2024-08-31 02:28:18.44594148 +0000 UTC m=+0.612025670 container start a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.449Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.449Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=arp 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=bcache 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=bonding 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=btrfs 2024-08-31T02:28:18.919 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=conntrack 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=cpu 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=diskstats 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=dmi 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=edac 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=entropy 2024-08-31T02:28:18.920 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=filefd 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=filesystem 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=hwmon 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=infiniband 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=ipvs 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=loadavg 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=mdadm 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=meminfo 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=netclass 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=netdev 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=netstat 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=nfs 2024-08-31T02:28:18.921 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=nfsd 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=nvme 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=os 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=pressure 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=rapl 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=schedstat 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=sockstat 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=softnet 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=stat 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=tapestats 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=textfile 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=time 2024-08-31T02:28:18.922 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=uname 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=vmstat 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=xfs 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:115 level=info collector=zfs 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[42621]: ts=2024-08-31T02:28:18.450Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 bash[42575]: a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a 2024-08-31T02:28:18.923 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:28:18 smithi102 systemd[1]: Started Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: Reconfiguring grafana.a (dependencies changed)... 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:19 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: Reconfiguring grafana.a (dependencies changed)... 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: Reconfiguring grafana.a (dependencies changed)... 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:19.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:19 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:20.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:20 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:20.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:20 smithi102 ceph-mon[13158]: Reconfiguring daemon grafana.a on smithi102 2024-08-31T02:28:20.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:20 smithi102 ceph-mon[13158]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:28:20.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:20.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[12201]: Reconfiguring daemon grafana.a on smithi102 2024-08-31T02:28:20.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[12201]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:28:20.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-31T02:28:20.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[17138]: Reconfiguring daemon grafana.a on smithi102 2024-08-31T02:28:20.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:20 smithi070 ceph-mon[17138]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:28:21.479 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 systemd[1]: Stopping Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:21.479 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 bash[42831]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana.a" found: no such container 2024-08-31T02:28:21.813 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[31398]: t=2024-08-31T02:28:21+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-08-31T02:28:21.813 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 podman[42842]: 2024-08-31 02:28:21.489141913 +0000 UTC m=+0.132338872 container died 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, release=236.1648460182, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, name=ubi8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-08-31T02:28:22.113 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 podman[42842]: 2024-08-31 02:28:21.911878846 +0000 UTC m=+0.555075762 container cleanup 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, vendor=Red Hat, Inc., description=Ceph Grafana Container, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, release=236.1648460182, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, name=ubi8, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.tags=base rhel8) 2024-08-31T02:28:22.113 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 bash[42842]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a 2024-08-31T02:28:22.113 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:21 smithi102 bash[42869]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana.a" found: no such container 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 podman[42854]: 2024-08-31 02:28:22.113203904 +0000 UTC m=+0.618473425 container remove 5683b141a5de637b441bfc9b09a6e3467fd2638e616286d1fe354fb1b2a5d3c1 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , description=Ceph Grafana Container, version=8.5, io.openshift.tags=base rhel8, name=ubi8, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=) 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 systemd[1]: Stopped Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 3.351s CPU time. 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 systemd[1]: Starting Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 podman[42931]: 2024-08-31 02:28:22.252026595 +0000 UTC m=+0.015056072 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-08-31T02:28:22.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 podman[42931]: 2024-08-31 02:28:22.355961729 +0000 UTC m=+0.118991196 container create 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, release=236.1648460182, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, vcs-type=git) 2024-08-31T02:28:22.762 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-mon[13158]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 podman[42931]: 2024-08-31 02:28:22.617142 +0000 UTC m=+0.380171465 container init 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Paul Cuzner , vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.24.2, io.openshift.expose-services=, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 podman[42931]: 2024-08-31 02:28:22.620119909 +0000 UTC m=+0.383149374 container start 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-03-28T10:36:18.413762, version=8.5, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.buildah.version=1.24.2, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vcs-type=git, release=236.1648460182) 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-08-31T02:28:22.763 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-08-31T02:28:22.764 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="App mode production" logger=settings 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=483.24µs 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 bash[42931]: 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 systemd[1]: Started Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-08-31T02:28:22.765 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-08-31T02:28:22.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:22 smithi070 ceph-mon[12201]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:28:22.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:22 smithi070 ceph-mon[17138]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:28:23.071 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:22] ENGINE Bus STOPPING 2024-08-31T02:28:23.071 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:28:23.072 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STOPPED 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-08-31T02:28:23.072 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:28:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:28:22+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-08-31T02:28:23.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STARTING 2024-08-31T02:28:23.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Serving on http://:::9283 2024-08-31T02:28:23.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STARTED 2024-08-31T02:28:23.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STOPPING 2024-08-31T02:28:23.763 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:23.503Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi102\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi102\", job=\"node\", machine=\"x86_64\", nodename=\"smithi102\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi102\", job=\"node\", machine=\"x86_64\", nodename=\"smithi102\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:23.763 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:28:23.763 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STOPPED 2024-08-31T02:28:23.763 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STARTING 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.764 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.765 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: Deploying daemon agent.smithi070 on smithi070 2024-08-31T02:28:23.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-mon[13158]: Deploying daemon agent.smithi102 on smithi102 2024-08-31T02:28:23.784 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070.front.sepia.ceph.com:9093"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:28:23.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102:3000"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: Deploying daemon agent.smithi070 on smithi070 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[12201]: Deploying daemon agent.smithi102 on smithi102 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: Deploying daemon agent.smithi070 on smithi070 2024-08-31T02:28:23.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:23 smithi070 ceph-mon[17138]: Deploying daemon agent.smithi102 on smithi102 2024-08-31T02:28:24.032 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Serving on http://:::9283 2024-08-31T02:28:24.032 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STARTED 2024-08-31T02:28:24.032 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:23] ENGINE Bus STOPPING 2024-08-31T02:28:24.564 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:24 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:28:24.564 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:24 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:24] ENGINE Bus STOPPED 2024-08-31T02:28:24.564 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:24 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:24] ENGINE Bus STARTING 2024-08-31T02:28:24.564 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:24 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:24] ENGINE Serving on http://:::9283 2024-08-31T02:28:24.564 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:24 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:28:24] ENGINE Bus STARTED 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:25.224 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:28:25.225 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:28:25.226 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:28:25.226 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.226 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:25.226 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:26.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:26 smithi102 ceph-mon[13158]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-08-31T02:28:26.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:26 smithi070 ceph-mon[12201]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-08-31T02:28:26.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:26 smithi070 ceph-mon[17138]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 871 B/s rd, 0 op/s 2024-08-31T02:28:28.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:28 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:28.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:28 smithi102 ceph-mon[13158]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:28.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:28.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:28.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:28.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[12201]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:28.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:28.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:28.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:28.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[17138]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:28.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:28.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:30.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:30 smithi070 ceph-mon[12201]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:30.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:30 smithi070 ceph-mon[17138]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:30 smithi102 ceph-mon[13158]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:31 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:31 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:32 smithi102 ceph-mon[13158]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:32.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:32 smithi070 ceph-mon[12201]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:32.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:32 smithi070 ceph-mon[17138]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:28:33] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-08-31T02:28:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:33.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:33.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:34 smithi102 ceph-mon[13158]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:34.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:34 smithi070 ceph-mon[12201]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:34.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:34 smithi070 ceph-mon[17138]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:36 smithi070 ceph-mon[12201]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:36 smithi070 ceph-mon[17138]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:36 smithi102 ceph-mon[13158]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:38 smithi102 ceph-mon[13158]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:38 smithi070 ceph-mon[12201]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:38 smithi070 ceph-mon[17138]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:38.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:40 smithi102 ceph-mon[13158]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:40 smithi070 ceph-mon[12201]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:40 smithi070 ceph-mon[17138]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:42.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:42 smithi102 ceph-mon[13158]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:42 smithi070 ceph-mon[12201]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:42 smithi070 ceph-mon[17138]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:43.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:28:43] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-08-31T02:28:43.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:44.569 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:44 smithi070 ceph-mon[12201]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:44.569 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:44 smithi070 ceph-mon[17138]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:44 smithi102 ceph-mon[13158]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:45 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:45 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:45 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:45 smithi102 ceph-mon[13158]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[12201]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:46.136 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:45 smithi070 ceph-mon[17138]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:47.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:28:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:48 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:48 smithi102 ceph-mon[13158]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[12201]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[17138]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:28:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:50.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:50 smithi102 ceph-mon[13158]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:50 smithi070 ceph-mon[12201]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:50 smithi070 ceph-mon[17138]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:52 smithi102 ceph-mon[13158]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:52 smithi070 ceph-mon[12201]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:52 smithi070 ceph-mon[17138]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:53.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:28:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:28:53] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-08-31T02:28:53.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:28:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:54 smithi102 ceph-mon[13158]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:54 smithi070 ceph-mon[12201]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:54 smithi070 ceph-mon[17138]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:56.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:56 smithi070 ceph-mon[12201]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:56.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:56 smithi070 ceph-mon[17138]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:56.470 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:56 smithi102 ceph-mon[13158]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:28:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:28:58 smithi102 ceph-mon[13158]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:28:58 smithi070 ceph-mon[12201]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:28:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:28:58 smithi070 ceph-mon[17138]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:28:58.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:28:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:28:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:00 smithi102 ceph-mon[13158]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:00 smithi070 ceph-mon[12201]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:00.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:00 smithi070 ceph-mon[17138]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:02 smithi102 ceph-mon[13158]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:02 smithi070 ceph-mon[12201]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:02.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:02 smithi070 ceph-mon[17138]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:03.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:03] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-08-31T02:29:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:04 smithi102 ceph-mon[13158]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:04 smithi070 ceph-mon[12201]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:04 smithi070 ceph-mon[17138]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:06.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:05 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:05 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:05 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:05 smithi102 ceph-mon[13158]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:06.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[12201]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:06.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:06.154 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:05 smithi070 ceph-mon[17138]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:07.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.516 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:08 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:08 smithi070 ceph-mon[12201]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:08 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:08 smithi070 ceph-mon[17138]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:08 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:08 smithi102 ceph-mon[13158]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:08.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:10 smithi070 ceph-mon[12201]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:10 smithi070 ceph-mon[17138]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:10 smithi102 ceph-mon[13158]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:12 smithi070 ceph-mon[12201]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:12 smithi070 ceph-mon[17138]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:12 smithi102 ceph-mon[13158]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:13.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:13] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-08-31T02:29:13.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:14 smithi070 ceph-mon[12201]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:14 smithi070 ceph-mon[17138]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:14 smithi102 ceph-mon[13158]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:16 smithi070 ceph-mon[12201]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:16 smithi070 ceph-mon[17138]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:16 smithi102 ceph-mon[13158]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:18 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:18 smithi102 ceph-mon[13158]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[12201]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[17138]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:18.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:20 smithi102 ceph-mon[13158]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:20 smithi070 ceph-mon[12201]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:20 smithi070 ceph-mon[17138]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:22 smithi102 ceph-mon[13158]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:22 smithi070 ceph-mon[12201]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:22 smithi070 ceph-mon[17138]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:23] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-08-31T02:29:23.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:24.643 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:24 smithi102 ceph-mon[13158]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:24 smithi070 ceph-mon[12201]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:24 smithi070 ceph-mon[17138]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.398 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:25.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:26.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:26 smithi070 ceph-mon[12201]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:26.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:26 smithi070 ceph-mon[17138]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:26 smithi102 ceph-mon[13158]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.615 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:28 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:28 smithi102 ceph-mon[13158]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:28.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:28 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:28 smithi070 ceph-mon[12201]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:28 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:28 smithi070 ceph-mon[17138]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:30.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:30 smithi102 ceph-mon[13158]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:30.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:30 smithi070 ceph-mon[12201]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:30.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:30 smithi070 ceph-mon[17138]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:32 smithi102 ceph-mon[13158]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:32.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:32 smithi070 ceph-mon[12201]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:32.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:32 smithi070 ceph-mon[17138]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:33] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-08-31T02:29:33.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:33.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:33.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:34 smithi102 ceph-mon[13158]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:34.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:34 smithi070 ceph-mon[12201]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:34.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:34 smithi070 ceph-mon[17138]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:36 smithi070 ceph-mon[12201]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:36 smithi070 ceph-mon[17138]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:36.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:36 smithi102 ceph-mon[13158]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:38 smithi102 ceph-mon[13158]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:38 smithi070 ceph-mon[12201]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:38 smithi070 ceph-mon[17138]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:38.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:40 smithi102 ceph-mon[13158]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:40 smithi070 ceph-mon[12201]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:40 smithi070 ceph-mon[17138]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:42 smithi102 ceph-mon[13158]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:42 smithi070 ceph-mon[12201]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:42 smithi070 ceph-mon[17138]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:43.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:43] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-08-31T02:29:43.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:44 smithi102 ceph-mon[13158]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:44 smithi070 ceph-mon[12201]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:44 smithi070 ceph-mon[17138]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[12201]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.423 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:46 smithi070 ceph-mon[17138]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:46.511 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.511 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.511 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:46.511 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:46 smithi102 ceph-mon[13158]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:47.777 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:47 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:47.777 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.777 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.777 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.777 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:47 smithi102 ceph-mon[13158]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[17138]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:29:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:47 smithi070 ceph-mon[12201]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:48.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:48.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:29:49.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:49 smithi070 ceph-mon[12201]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:49.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:49 smithi070 ceph-mon[17138]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:49 smithi102 ceph-mon[13158]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:52 smithi102 ceph-mon[13158]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:52 smithi070 ceph-mon[12201]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:52 smithi070 ceph-mon[17138]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:53.919 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:29:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:29:53] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-08-31T02:29:54.488 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:29:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:54 smithi102 ceph-mon[13158]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:54 smithi070 ceph-mon[12201]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:54 smithi070 ceph-mon[17138]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:29:56.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:56 smithi070 ceph-mon[12201]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:56.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:56 smithi070 ceph-mon[17138]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:56.470 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:56 smithi102 ceph-mon[13158]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:29:58 smithi102 ceph-mon[13158]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:29:58 smithi070 ceph-mon[12201]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:29:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:29:58 smithi070 ceph-mon[17138]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:29:58.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:29:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:29:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:00 smithi102 ceph-mon[13158]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:00 smithi102 ceph-mon[13158]: overall HEALTH_OK 2024-08-31T02:30:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:00 smithi070 ceph-mon[12201]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:00 smithi070 ceph-mon[12201]: overall HEALTH_OK 2024-08-31T02:30:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:00 smithi070 ceph-mon[17138]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:00 smithi070 ceph-mon[17138]: overall HEALTH_OK 2024-08-31T02:30:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:02 smithi102 ceph-mon[13158]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:02 smithi070 ceph-mon[12201]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:02.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:02 smithi070 ceph-mon[17138]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:03.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:03] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:30:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:04 smithi102 ceph-mon[13158]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:04 smithi070 ceph-mon[12201]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:04 smithi070 ceph-mon[17138]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[12201]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[17138]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.399 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.400 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.506 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:06 smithi102 ceph-mon[13158]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:06.506 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.506 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:06.506 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:07 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:07 smithi102 ceph-mon[13158]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[12201]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:08.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:07 smithi070 ceph-mon[17138]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:08.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:10 smithi102 ceph-mon[13158]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:10 smithi070 ceph-mon[12201]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:10 smithi070 ceph-mon[17138]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:12 smithi102 ceph-mon[13158]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:12 smithi070 ceph-mon[12201]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:12 smithi070 ceph-mon[17138]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:13.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:13] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:30:13.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:14 smithi102 ceph-mon[13158]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:14 smithi070 ceph-mon[12201]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:14 smithi070 ceph-mon[17138]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:16.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:16 smithi070 ceph-mon[12201]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:16.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:16 smithi070 ceph-mon[17138]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:16 smithi102 ceph-mon[13158]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:18 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:18 smithi102 ceph-mon[13158]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[12201]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[17138]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:18.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:20 smithi102 ceph-mon[13158]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:20 smithi070 ceph-mon[12201]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:20 smithi070 ceph-mon[17138]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:22 smithi102 ceph-mon[13158]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:22 smithi070 ceph-mon[12201]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:22 smithi070 ceph-mon[17138]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:23] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-08-31T02:30:23.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:24 smithi102 ceph-mon[13158]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:24 smithi070 ceph-mon[12201]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:24 smithi070 ceph-mon[17138]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:25 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:25 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:25.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:25 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[12201]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[17138]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:26 smithi102 ceph-mon[13158]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:28.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[12201]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:27 smithi070 ceph-mon[17138]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:28.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:27 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:28.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:28.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:27 smithi102 ceph-mon[13158]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:28.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:30 smithi102 ceph-mon[13158]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:30 smithi070 ceph-mon[12201]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:30 smithi070 ceph-mon[17138]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:32 smithi102 ceph-mon[13158]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:32 smithi070 ceph-mon[12201]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:32 smithi070 ceph-mon[17138]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:33.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:33] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:30:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:34 smithi102 ceph-mon[13158]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:34 smithi070 ceph-mon[12201]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:34 smithi070 ceph-mon[17138]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:36 smithi070 ceph-mon[12201]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:36.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:36 smithi070 ceph-mon[17138]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:36 smithi102 ceph-mon[13158]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:38.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:38.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:38 smithi102 ceph-mon[13158]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:38 smithi070 ceph-mon[12201]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:38 smithi070 ceph-mon[17138]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:38.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:40 smithi102 ceph-mon[13158]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:40 smithi070 ceph-mon[17138]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:40 smithi070 ceph-mon[12201]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:30:42.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:42 smithi102 ceph-mon[13158]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:42 smithi070 ceph-mon[12201]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:42 smithi070 ceph-mon[17138]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:30:43.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:43] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:30:43.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:44 smithi102 ceph-mon[13158]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:44 smithi070 ceph-mon[12201]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:44 smithi070 ceph-mon[17138]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:46 smithi102 ceph-mon[13158]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[12201]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:46.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[17138]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:48.234 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:48.345 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:48.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:48.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:48.346 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:30:48.921 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:50.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-08-31T02:30:50.621 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:50 smithi102 ceph-mon[13158]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:50.694 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:50 smithi070 ceph-mon[12201]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:50.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:50 smithi070 ceph-mon[17138]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:52.471 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:52 smithi070 ceph-mon[12201]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:30:52.471 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:52 smithi070 ceph-mon[17138]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:30:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:52 smithi102 ceph-mon[13158]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:30:53.080 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled to redeploy mgr.y on host 'smithi070' 2024-08-31T02:30:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1147648148' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1147648148' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1147648148' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:53 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:30:53.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps --refresh' 2024-08-31T02:30:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:30:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:30:53] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:30:54.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[12201]: from='client.25051 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:54.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[12201]: Schedule redeploy daemon mgr.y 2024-08-31T02:30:54.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[12201]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:54.396 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[17138]: from='client.25051 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:54.396 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[17138]: Schedule redeploy daemon mgr.y 2024-08-31T02:30:54.397 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:54 smithi070 ceph-mon[17138]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:54 smithi102 ceph-mon[13158]: from='client.25051 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:54 smithi102 ceph-mon[13158]: Schedule redeploy daemon mgr.y 2024-08-31T02:30:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:54 smithi102 ceph-mon[13158]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:30:56.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: Deploying daemon mgr.y on smithi070 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[12201]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:30:56.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.145 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: Deploying daemon mgr.y on smithi070 2024-08-31T02:30:56.145 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:56 smithi070 ceph-mon[17138]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:56.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:30:56.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:30:56.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: Deploying daemon mgr.y on smithi070 2024-08-31T02:30:56.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:56 smithi102 ceph-mon[13158]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:57.933 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:57 smithi070 systemd[1]: Stopping Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:30:57.933 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:30:57.933 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 12s ago 16m - - 2024-08-31T02:30:57.933 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 10s ago 16m - - 2024-08-31T02:30:57.933 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 12s ago 11m 26.0M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:30:57.933 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (2m) 10s ago 11m 46.4M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (2m) 12s ago 10m 51.1M - 3.5 e1d6a67b021e 97aaa626628a 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (5m) 10s ago 15m 509M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:9283 running (17m) 12s ago 17m 411M - 17.2.0 e1d6a67b021e 4a3942799177 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (17m) 12s ago 17m 62.0M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (16m) 10s ago 16m 48.2M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (16m) 12s ago 16m 46.3M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 12s ago 11m 17.7M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (2m) 10s ago 11m 17.2M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (15m) 12s ago 15m 55.0M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (15m) 12s ago 15m 56.0M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (14m) 12s ago 14m 49.1M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:30:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (14m) 12s ago 14m 51.2M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (13m) 10s ago 13m 52.7M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (13m) 10s ago 13m 49.8M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (13m) 10s ago 13m 51.5M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (12m) 10s ago 12m 55.4M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 10s ago 12m 69.8M - 2.33.4 514e6a882f6e c7c13a49447d 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (10m) 12s ago 10m 74.9M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:30:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (10m) 10s ago 10m 73.3M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:30:58.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:58 smithi070 podman[56639]: 2024-08-31 02:30:58.159325206 +0000 UTC m=+0.309726685 container died 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, distribution-scope=public, GIT_BRANCH=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.component=centos-stream-container) 2024-08-31T02:30:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:58 smithi102 ceph-mon[13158]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:58 smithi070 ceph-mon[12201]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:30:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:58 smithi070 ceph-mon[17138]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:30:58.733 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:30:58.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:30:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:30:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:30:58.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:58 smithi070 podman[56639]: 2024-08-31 02:30:58.944928549 +0000 UTC m=+1.095330018 container cleanup 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, io.openshift.expose-services=, name=centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, ceph=True) 2024-08-31T02:30:58.964 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:58 smithi070 bash[56639]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y 2024-08-31T02:30:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:30:59 smithi070 ceph-mon[12201]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:59.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:59 smithi070 podman[56664]: 2024-08-31 02:30:59.154910368 +0000 UTC m=+1.169406766 container remove 4a39427991770a6c2a94661c8d36c61455d99a486ce1a351b8cf261270182e9e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux ) 2024-08-31T02:30:59.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:59 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:30:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:30:59 smithi070 ceph-mon[17138]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:30:59 smithi102 ceph-mon[13158]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:30:59.945 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:30:59 smithi070 podman[56713]: 2024-08-31 02:30:59.60858368 +0000 UTC m=+0.636514164 container died 6cf96ea8ca306d5fb014b6bc57e01f42146753f25a6e049be866833eb1cc583f (image=quay.io/ceph/ceph:v17.2.0, name=angry_bassi, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , architecture=x86_64, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:31:00.487 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:00 smithi070 ceph-mon[12201]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:31:00.487 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:00 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-31T02:31:00.487 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:00 smithi070 systemd[1]: Stopped Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:31:00.487 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:00 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service: Consumed 1min 10.056s CPU time. 2024-08-31T02:31:00.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:00 smithi070 ceph-mon[17138]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:31:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:00 smithi102 ceph-mon[13158]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:31:00.824 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:00 smithi070 systemd[1]: Starting Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:31:01.131 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 podman[56886]: 2024-08-31 02:31:01.032917541 +0000 UTC m=+0.110997009 container create 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-08-31T02:31:01.131 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 podman[56886]: 2024-08-31 02:31:00.937573956 +0000 UTC m=+0.015653435 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 podman[56886]: 2024-08-31 02:31:01.244103528 +0000 UTC m=+0.322183004 container init 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 podman[56886]: 2024-08-31 02:31:01.247040132 +0000 UTC m=+0.325119612 container start 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-34b9d27, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-mgr[56919]: -- 172.21.15.70:0/2536810666 <== mon.1 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55af5164d0e0 con 0x55af5162a800 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 bash[56886]: 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 systemd[1]: Started Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:01.369+0000 7fac37501040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:31:01.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:01.441+0000 7fac37501040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:31:01.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:01.622+0000 7fac37501040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:31:01.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:01.696+0000 7fac37501040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:31:02.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:01.862+0000 7fac37501040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:31:02.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:02.006+0000 7fac37501040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:31:02.602 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:02.332+0000 7fac37501040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:31:02.602 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:02.410+0000 7fac37501040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.603 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:31:02.604 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.869 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:02.601+0000 7fac37501040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:31:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:03.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:03.679 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:03.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:03.577+0000 7fac37501040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:31:03.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:03.656+0000 7fac37501040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:31:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:03] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:31:04.673 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-mon[12201]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:04.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-mon[17138]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:04.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.406+0000 7fac37501040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:31:04.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.528+0000 7fac37501040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:31:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:04 smithi102 ceph-mon[13158]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:04.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.676+0000 7fac37501040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:31:04.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.738+0000 7fac37501040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:31:04.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.890+0000 7fac37501040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:31:05.263 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:04.958+0000 7fac37501040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:31:05.263 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.025+0000 7fac37501040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:31:05.263 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.197+0000 7fac37501040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:31:05.263 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.262+0000 7fac37501040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:31:05.515 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.334+0000 7fac37501040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:31:05.515 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.420+0000 7fac37501040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:31:05.515 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.481+0000 7fac37501040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:31:05.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:05.543+0000 7fac37501040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:31:06.478 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[12201]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:06.478 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.478 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.478 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[17138]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:06.369+0000 7fac37501040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:31:06] ENGINE Bus STARTING 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: CherryPy Checker: 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: The Application mounted at '' has an empty config. 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:31:06.479 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:31:06] ENGINE Serving on http://:::9283 2024-08-31T02:31:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:06 smithi102 ceph-mon[13158]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:06.743 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:31:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:31:06] ENGINE Bus STARTED 2024-08-31T02:31:07.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: Standby manager daemon y restarted 2024-08-31T02:31:07.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: Standby manager daemon y started 2024-08-31T02:31:07.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: Standby manager daemon y restarted 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: Standby manager daemon y started 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:31:07.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:31:07.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:31:07.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:31:07.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: Standby manager daemon y restarted 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: Standby manager daemon y started 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:31:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:07.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:08.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: mgrmap e27: x(active, since 3m), standbys: y 2024-08-31T02:31:08.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-mon[13158]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: mgrmap e27: x(active, since 3m), standbys: y 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[17138]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: mgrmap e27: x(active, since 3m), standbys: y 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:08.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:08 smithi070 ceph-mon[12201]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:08.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:10.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:10 smithi070 ceph-mon[12201]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:10.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:10 smithi070 ceph-mon[17138]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:10 smithi102 ceph-mon[13158]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:12 smithi102 ceph-mon[13158]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:12 smithi070 ceph-mon[12201]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:12 smithi070 ceph-mon[17138]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:13.920 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:13] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:31:13.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:14 smithi102 ceph-mon[13158]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:14 smithi070 ceph-mon[12201]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:14 smithi070 ceph-mon[17138]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:16 smithi102 ceph-mon[13158]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:16 smithi070 ceph-mon[12201]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:16 smithi070 ceph-mon[17138]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:18 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:18.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:18 smithi102 ceph-mon[13158]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:18.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[12201]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[17138]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:18.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:20 smithi102 ceph-mon[13158]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:20 smithi070 ceph-mon[12201]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:20.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:20 smithi070 ceph-mon[17138]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:22 smithi102 ceph-mon[13158]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:22 smithi070 ceph-mon[12201]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:22 smithi070 ceph-mon[17138]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:23] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-08-31T02:31:23.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:24 smithi102 ceph-mon[13158]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:24 smithi070 ceph-mon[12201]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:24 smithi070 ceph-mon[17138]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:26.470 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:26 smithi102 ceph-mon[13158]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:26.470 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[12201]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[17138]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:28.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-mon[13158]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:28.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.620 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[12201]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[17138]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:28.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:30.411 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:30 smithi070 ceph-mon[12201]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:30.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:30 smithi070 ceph-mon[17138]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:30 smithi102 ceph-mon[13158]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:32.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:32 smithi070 ceph-mon[12201]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:32.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:32 smithi070 ceph-mon[17138]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:32 smithi102 ceph-mon[13158]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:33.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:33] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-08-31T02:31:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:34 smithi102 ceph-mon[13158]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:34 smithi070 ceph-mon[12201]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:34 smithi070 ceph-mon[17138]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:36.501 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:36 smithi070 ceph-mon[12201]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:36.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:36 smithi070 ceph-mon[17138]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:36 smithi102 ceph-mon[13158]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:38 smithi102 ceph-mon[13158]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:38 smithi070 ceph-mon[12201]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:38 smithi070 ceph-mon[17138]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:38.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:40 smithi102 ceph-mon[13158]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:40 smithi070 ceph-mon[12201]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:40 smithi070 ceph-mon[17138]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:42.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:42 smithi102 ceph-mon[13158]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:42 smithi070 ceph-mon[12201]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:42 smithi070 ceph-mon[17138]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:43.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:43] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-08-31T02:31:43.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:44 smithi102 ceph-mon[13158]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:44 smithi070 ceph-mon[12201]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:44 smithi070 ceph-mon[17138]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:46 smithi102 ceph-mon[13158]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:46 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[12201]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[17138]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:46 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:48.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:48.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:48.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:31:48.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:31:50.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:50 smithi102 ceph-mon[13158]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:50.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:50 smithi070 ceph-mon[12201]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:50.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:50 smithi070 ceph-mon[17138]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:52 smithi102 ceph-mon[13158]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:52.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:52 smithi070 ceph-mon[12201]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:52.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:52 smithi070 ceph-mon[17138]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:53.709 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:31:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:31:53] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:31:53.709 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:53.503Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:31:54.917 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:54 smithi102 ceph-mon[13158]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:54.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:54 smithi070 ceph-mon[12201]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:54.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:54 smithi070 ceph-mon[17138]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:31:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:56 smithi102 ceph-mon[13158]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:56 smithi070 ceph-mon[12201]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:56 smithi070 ceph-mon[17138]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:31:58 smithi102 ceph-mon[13158]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:31:58 smithi070 ceph-mon[12201]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:31:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:31:58 smithi070 ceph-mon[17138]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:31:58.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:31:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:31:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:00 smithi102 ceph-mon[13158]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:00 smithi070 ceph-mon[12201]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:00 smithi070 ceph-mon[17138]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:02 smithi102 ceph-mon[13158]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:32:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:32:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:32:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[12201]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:32:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[17138]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:32:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:03.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:03] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:32:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:04 smithi102 ceph-mon[13158]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:04 smithi070 ceph-mon[12201]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:04 smithi070 ceph-mon[17138]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[17138]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[12201]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:06.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:06 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:06 smithi102 ceph-mon[13158]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:06 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-mon[13158]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[17138]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[12201]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:10 smithi102 ceph-mon[13158]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:10 smithi070 ceph-mon[12201]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:10.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:10 smithi070 ceph-mon[17138]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:12 smithi102 ceph-mon[13158]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:12 smithi070 ceph-mon[12201]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:12.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:12 smithi070 ceph-mon[17138]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:13.583 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:13.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:13.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:13] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:32:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:14 smithi102 ceph-mon[13158]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:14.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:14 smithi070 ceph-mon[12201]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:14.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:14 smithi070 ceph-mon[17138]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:16 smithi102 ceph-mon[13158]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:16 smithi070 ceph-mon[12201]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:16 smithi070 ceph-mon[17138]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:18 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:18 smithi102 ceph-mon[13158]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:18.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[12201]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[17138]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:18.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:20 smithi102 ceph-mon[13158]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:20 smithi070 ceph-mon[12201]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:20 smithi070 ceph-mon[17138]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:22 smithi102 ceph-mon[13158]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:22 smithi070 ceph-mon[12201]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:22 smithi070 ceph-mon[17138]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:23] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-08-31T02:32:23.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:23.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:24.635 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:24 smithi070 ceph-mon[12201]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:24.635 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:24 smithi070 ceph-mon[17138]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:24.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:24 smithi102 ceph-mon[13158]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:26 smithi102 ceph-mon[13158]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.471 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:26 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[12201]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[17138]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:26 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:28.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-mon[13158]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:28.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[12201]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[17138]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:28.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:30 smithi102 ceph-mon[13158]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:30 smithi070 ceph-mon[12201]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:30 smithi070 ceph-mon[17138]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:32 smithi070 ceph-mon[12201]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:32.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:32 smithi070 ceph-mon[17138]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:32 smithi102 ceph-mon[13158]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:33.505 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:33.531 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:33.531 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:33] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:32:33.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:33.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:34 smithi102 ceph-mon[13158]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:34 smithi070 ceph-mon[12201]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:34 smithi070 ceph-mon[17138]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:36.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:36 smithi070 ceph-mon[12201]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:36.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:36 smithi070 ceph-mon[17138]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:36 smithi102 ceph-mon[13158]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:38.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:38 smithi102 ceph-mon[13158]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:38.680 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:38.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:38 smithi070 ceph-mon[12201]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:38.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:38.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:38 smithi070 ceph-mon[17138]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:38.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:40 smithi102 ceph-mon[13158]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:40 smithi070 ceph-mon[12201]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:40 smithi070 ceph-mon[17138]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:42 smithi102 ceph-mon[13158]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:42 smithi070 ceph-mon[12201]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:42 smithi070 ceph-mon[17138]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:43.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:43] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:32:43.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:43.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:44 smithi102 ceph-mon[13158]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:44 smithi070 ceph-mon[12201]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:44 smithi070 ceph-mon[17138]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:46 smithi102 ceph-mon[13158]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:46 smithi070 ceph-mon[12201]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:46 smithi070 ceph-mon[17138]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:48 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:32:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:32:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:50 smithi102 ceph-mon[13158]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:50 smithi070 ceph-mon[12201]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:50 smithi070 ceph-mon[17138]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:52.545 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:52 smithi102 ceph-mon[13158]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:52 smithi070 ceph-mon[12201]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:52 smithi070 ceph-mon[17138]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:53.599 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:53.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:32:53.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:32:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:32:53] "GET /metrics HTTP/1.1" 200 37897 "" "Prometheus/2.33.4" 2024-08-31T02:32:54.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:54 smithi070 ceph-mon[12201]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:54.511 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:54 smithi070 ceph-mon[17138]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:54 smithi102 ceph-mon[13158]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:32:56.480 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:56 smithi102 ceph-mon[13158]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:56 smithi070 ceph-mon[12201]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:56.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:56 smithi070 ceph-mon[17138]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:58.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:32:58 smithi102 ceph-mon[13158]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:32:58 smithi070 ceph-mon[12201]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:32:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:32:58 smithi070 ceph-mon[17138]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:32:58.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:32:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:32:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"172.21.15.102:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", cluster_addr=\"172.21.15.70\", device_class=\"ssd\", hostname=\"smithi070\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.70\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:33:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:00 smithi102 ceph-mon[13158]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:00 smithi070 ceph-mon[12201]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:00 smithi070 ceph-mon[17138]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:02 smithi102 ceph-mon[13158]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:33:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:33:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:33:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[12201]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:33:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:33:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:33:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[17138]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:33:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:33:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:33:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:03.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:33:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:33:03.504Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi070\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"862b0c56-673e-11ef-bcd4-c7b262605968\", domainname=\"(none)\", instance=\"smithi070\", job=\"node\", machine=\"x86_64\", nodename=\"smithi070\", release=\"5.14.0-503.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-31T02:33:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:03.966 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:03] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-08-31T02:33:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:04 smithi102 ceph-mon[13158]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:04 smithi070 ceph-mon[12201]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:04.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:04 smithi070 ceph-mon[17138]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:06.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:06 smithi102 ceph-mon[13158]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:06 smithi070 ceph-mon[12201]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:06 smithi070 ceph-mon[17138]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:07.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.855 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.855 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:08 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:08 smithi102 ceph-mon[13158]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:08 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[12201]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[17138]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:08 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:10 smithi102 ceph-mon[13158]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:10 smithi070 ceph-mon[12201]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:10.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:10 smithi070 ceph-mon[17138]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:12 smithi102 ceph-mon[13158]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:12 smithi070 ceph-mon[12201]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:12.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:12 smithi070 ceph-mon[17138]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:13] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-08-31T02:33:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:14 smithi070 ceph-mon[17138]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:14 smithi070 ceph-mon[12201]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:14 smithi102 ceph-mon[13158]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:16 smithi102 ceph-mon[13158]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:16 smithi070 ceph-mon[17138]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:16 smithi070 ceph-mon[12201]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:18 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:18 smithi102 ceph-mon[13158]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:18 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[12201]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[17138]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:18.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:18 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:20 smithi102 ceph-mon[13158]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:20 smithi070 ceph-mon[12201]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:20 smithi070 ceph-mon[17138]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:22 smithi102 ceph-mon[13158]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:22 smithi070 ceph-mon[12201]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:22 smithi070 ceph-mon[17138]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:24.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:23] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:33:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:24 smithi102 ceph-mon[13158]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:24 smithi070 ceph-mon[12201]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:24 smithi070 ceph-mon[17138]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:26.605 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:26 smithi070 ceph-mon[12201]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:26.605 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:26 smithi070 ceph-mon[17138]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:26 smithi102 ceph-mon[13158]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:27 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:27.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:27 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:28 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:28 smithi102 ceph-mon[13158]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:28 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[12201]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[17138]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:28.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:28 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:30.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:30 smithi102 ceph-mon[13158]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:30.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:30 smithi070 ceph-mon[12201]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:30.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:30 smithi070 ceph-mon[17138]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:32 smithi102 ceph-mon[13158]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:32.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:32 smithi070 ceph-mon[12201]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:32.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:32 smithi070 ceph-mon[17138]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:33.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:33 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:33.775 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:33] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:33:33.922 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:33 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:33.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:33 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:34 smithi102 ceph-mon[13158]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:34.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:34 smithi070 ceph-mon[12201]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:34.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:34 smithi070 ceph-mon[17138]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:36.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:36 smithi070 ceph-mon[12201]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:36.318 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:36 smithi070 ceph-mon[17138]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:36 smithi102 ceph-mon[13158]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:38 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:38 smithi102 ceph-mon[13158]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:38 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:38.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:38 smithi070 ceph-mon[12201]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:38.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:38 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:38.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:38 smithi070 ceph-mon[17138]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:40 smithi102 ceph-mon[13158]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:40 smithi070 ceph-mon[12201]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:40 smithi070 ceph-mon[17138]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:42 smithi102 ceph-mon[13158]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:42 smithi070 ceph-mon[12201]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:42 smithi070 ceph-mon[17138]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:44.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:43] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:33:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:44 smithi102 ceph-mon[13158]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:44 smithi070 ceph-mon[12201]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:44 smithi070 ceph-mon[17138]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:46 smithi102 ceph-mon[13158]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:46 smithi070 ceph-mon[12201]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:46 smithi070 ceph-mon[17138]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:47 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:47 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:33:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:49 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:49 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:33:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:50 smithi102 ceph-mon[13158]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:50 smithi070 ceph-mon[12201]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:50 smithi070 ceph-mon[17138]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:52 smithi102 ceph-mon[13158]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:52 smithi070 ceph-mon[12201]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:52 smithi070 ceph-mon[17138]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:53.996 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:33:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:33:53] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-08-31T02:33:54.572 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:54 smithi102 ceph-mon[13158]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:54 smithi070 ceph-mon[12201]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:54 smithi070 ceph-mon[17138]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:33:56.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:56 smithi070 ceph-mon[12201]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:56.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:56 smithi070 ceph-mon[17138]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:56 smithi102 ceph-mon[13158]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:58 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:33:58 smithi102 ceph-mon[13158]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:58 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:33:58 smithi070 ceph-mon[12201]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:33:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:58 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:33:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:33:58 smithi070 ceph-mon[17138]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:00 smithi102 ceph-mon[13158]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:00 smithi070 ceph-mon[12201]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:00 smithi070 ceph-mon[17138]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:01.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:34:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:02 smithi102 ceph-mon[13158]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:02 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:02 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[12201]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[17138]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:02 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:03.432 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:34:03.432 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 16s ago 19m - - 2024-08-31T02:34:03.433 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 15s ago 19m - - 2024-08-31T02:34:03.433 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (6m) 16s ago 14m 24.5M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:34:03.433 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (5m) 15s ago 14m 53.1M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:34:03.433 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (5m) 16s ago 13m 53.8M - 3.5 e1d6a67b021e 97aaa626628a 2024-08-31T02:34:03.433 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (8m) 15s ago 18m 513M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (3m) 16s ago 20m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (20m) 16s ago 20m 66.7M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (19m) 15s ago 19m 51.2M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (19m) 16s ago 19m 48.4M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6m) 16s ago 14m 16.7M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:34:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (5m) 15s ago 14m 16.7M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (18m) 16s ago 18m 55.9M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (18m) 16s ago 18m 57.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (17m) 16s ago 17m 49.5M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (17m) 16s ago 17m 52.3M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (16m) 15s ago 16m 53.5M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (16m) 15s ago 16m 49.9M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:34:03.435 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (16m) 15s ago 16m 51.7M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:34:03.436 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (15m) 15s ago 15m 55.5M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:34:03.436 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (5m) 15s ago 15m 72.1M - 2.33.4 514e6a882f6e c7c13a49447d 2024-08-31T02:34:03.436 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (13m) 16s ago 13m 75.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:34:03.436 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (13m) 15s ago 13m 74.5M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:34:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:03 smithi102 ceph-mon[13158]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:03 smithi070 ceph-mon[12201]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:03 smithi070 ceph-mon[17138]: from='mgr.34790 172.21.15.102:0/2776327799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:04.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions' 2024-08-31T02:34:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:03] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:34:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:04 smithi102 ceph-mon[13158]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:04 smithi102 ceph-mon[13158]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:34:04.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:04 smithi070 ceph-mon[12201]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:04.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:04 smithi070 ceph-mon[12201]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:34:04.690 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:04 smithi070 ceph-mon[17138]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:04.690 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:04 smithi070 ceph-mon[17138]: from='client.15162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:34:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:34:06.249 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:34:06.502 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:06 smithi070 ceph-mon[12201]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:06.503 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:06 smithi070 ceph-mon[17138]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:06 smithi102 ceph-mon[13158]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:06.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:34:07.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3013343875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:34:07.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3013343875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:34:07.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:07 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:07 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3013343875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:34:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:07 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:08.506 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:08 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:08.506 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:08 smithi070 ceph-mon[12201]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:08.507 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:08 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:08.507 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:08 smithi070 ceph-mon[17138]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:08 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:08 smithi102 ceph-mon[13158]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:08.980 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_OK 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[12201]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/849396758' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[17138]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.551 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:09 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/849396758' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:34:09.574 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph -s' 2024-08-31T02:34:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:09 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:09 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:09 smithi102 ceph-mon[13158]: from='mgr.34790 ' entity='mgr.x' 2024-08-31T02:34:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:09 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/849396758' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:34:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:10 smithi070 ceph-mon[12201]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:10.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:10 smithi070 ceph-mon[17138]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:10 smithi102 ceph-mon[13158]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: cluster: 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: id: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: health: HEALTH_OK 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: services: 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: mon: 3 daemons, quorum a,c,b (age 19m) 2024-08-31T02:34:11.611 INFO:teuthology.orchestra.run.smithi070.stdout: mgr: x(active, since 6m), standbys: y 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 16m) 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: data: 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: pools: 6 pools, 161 pgs 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: objects: 241 objects, 585 KiB 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: pgs: 161 active+clean 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: io: 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-08-31T02:34:11.612 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:34:12.297 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph mgr fail' 2024-08-31T02:34:12.590 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:12 smithi070 ceph-mon[12201]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:12.591 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:12 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3320779023' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:34:12.591 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:12 smithi070 ceph-mon[17138]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:12.591 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:12 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3320779023' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:34:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:12 smithi102 ceph-mon[13158]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:12 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3320779023' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:34:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:13] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-mon[13158]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2849437980' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-mon[13158]: osdmap e105: 8 total, 8 up, 8 in 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.389+0000 7fc66076a640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-31T02:34:14.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ignoring --setuser ceph since I am not root 2024-08-31T02:34:14.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ignoring --setgroup ceph since I am not root 2024-08-31T02:34:14.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-mgr[37911]: -- 172.21.15.102:0/2587009674 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5589d8aed0e0 con 0x5589d8acb000 2024-08-31T02:34:14.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.538+0000 7fd138777040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:34:14.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.594+0000 7fd138777040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:34:14.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE Bus STOPPING 2024-08-31T02:34:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[12201]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2849437980' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[12201]: osdmap e105: 8 total, 8 up, 8 in 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[17138]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2849437980' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:34:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-mon[17138]: osdmap e105: 8 total, 8 up, 8 in 2024-08-31T02:34:15.058 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.745+0000 7fd138777040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:34:15.058 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.816+0000 7fd138777040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:34:15.058 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:14.970+0000 7fd138777040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:34:15.071 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:34:15.071 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE Bus STOPPED 2024-08-31T02:34:15.071 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE Bus STARTING 2024-08-31T02:34:15.071 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE Serving on http://:::9283 2024-08-31T02:34:15.071 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:14] ENGINE Bus STARTED 2024-08-31T02:34:15.101 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:34:15.385 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:15.119+0000 7fd138777040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:34:15.386 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:15.082Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:34:15.386 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:15.083Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:34:15.386 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:15.084Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:34:15.386 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:15.084Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:34:15.386 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:15.085Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:34:15.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:34:15.386 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: mgrmap e28: y(active, starting, since 0.149291s) 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:34:15.387 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:34:15.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:34:15.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:34:15.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:15.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:34:15.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:34:15.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:15.441+0000 7fd138777040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:15.502+0000 7fd138777040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: mgrmap e28: y(active, starting, since 0.149291s) 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:34:15.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: Manager daemon y is now available 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:34:15.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: mgrmap e28: y(active, starting, since 0.149291s) 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:34:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: Manager daemon y is now available 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:34:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:34:16.053 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:15.691+0000 7fd138777040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:34:16.056+0000 7fab8e4b1640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:34:16] ENGINE Error in HTTPServer.serve 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._connections.run(self.expiration_interval) 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:34:16.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._run(expiration_interval) 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s = self.context.wrap_socket( 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.sslsocket_class._create( 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.do_handshake() 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._sslobj.do_handshake() 2024-08-31T02:34:16.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:34:16.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: mgrmap e29: y(active, since 1.15786s) 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: [31/Aug/2024:02:34:15] ENGINE Bus STARTING 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: [31/Aug/2024:02:34:15] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: [31/Aug/2024:02:34:16] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: [31/Aug/2024:02:34:16] ENGINE Bus STARTED 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: [31/Aug/2024:02:34:16] ENGINE Error in HTTPServer.serve 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:34:16.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:34:16.623 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: mgrmap e29: y(active, since 1.15786s) 2024-08-31T02:34:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: [31/Aug/2024:02:34:15] ENGINE Bus STARTING 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: [31/Aug/2024:02:34:15] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: [31/Aug/2024:02:34:16] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: [31/Aug/2024:02:34:16] ENGINE Bus STARTED 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: [31/Aug/2024:02:34:16] ENGINE Error in HTTPServer.serve 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:34:16.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:34:16.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: mgrmap e29: y(active, since 1.15786s) 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: [31/Aug/2024:02:34:15] ENGINE Bus STARTING 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: [31/Aug/2024:02:34:15] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: [31/Aug/2024:02:34:16] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: [31/Aug/2024:02:34:16] ENGINE Bus STARTED 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: [31/Aug/2024:02:34:16] ENGINE Error in HTTPServer.serve 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:34:16.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:34:16.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:16 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:16.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:16.620+0000 7fd138777040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:34:16.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:16.692+0000 7fd138777040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:34:17.585 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:17.454+0000 7fd138777040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:34:17.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: Adjusting osd_memory_target on smithi070 to 2736M 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:17.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:34:17.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:34:17.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:17.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: Adjusting osd_memory_target on smithi070 to 2736M 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:34:17.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='client.34853 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:34:17.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: Adjusting osd_memory_target on smithi070 to 2736M 2024-08-31T02:34:17.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:17.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:17.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:17.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:34:17.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:17 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:34:17.843 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:17.584+0000 7fd138777040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:34:17.843 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:17.790+0000 7fd138777040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:34:18.130 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:17.872+0000 7fd138777040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:34:18.130 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.065+0000 7fd138777040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:34:18.390 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.151+0000 7fd138777040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:34:18.390 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.221+0000 7fd138777040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.406+0000 7fd138777040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.475+0000 7fd138777040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.546+0000 7fd138777040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.630+0000 7fd138777040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: mgrmap e30: y(active, since 3s) 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:18.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:18.784 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.784 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: mgrmap e30: y(active, since 3s) 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.785 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.786 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:18.787 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: mgrmap e30: y(active, since 3s) 2024-08-31T02:34:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.787 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:34:18.788 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:18.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:18.789 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:18 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:19.036 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.692+0000 7fd138777040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:34:19.036 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:18.752+0000 7fd138777040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:19.601+0000 7fd138777040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:34:19] ENGINE Bus STARTING 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: CherryPy Checker: 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: The Application mounted at '' has an empty config. 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:34:19] ENGINE Serving on http://:::9283 2024-08-31T02:34:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:34:19] ENGINE Bus STARTED 2024-08-31T02:34:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:34:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: mgrmap e31: y(active, since 5s) 2024-08-31T02:34:20.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:34:20.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:34:20.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:34:20.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:34:20.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:20 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:34:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:34:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:20.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: mgrmap e31: y(active, since 5s) 2024-08-31T02:34:20.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:34:20.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:34:20.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:34:20.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:34:20.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:34:20.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:34:20.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:20.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:20.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:20.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:20.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: mgrmap e31: y(active, since 5s) 2024-08-31T02:34:20.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:34:20.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:34:20.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:34:20.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:34:20.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:20 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:34:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:21 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.848 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:34:21.848 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-31T02:34:21.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:34:21.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:21.849 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:21 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-mon[12201]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-mon[17138]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:34:23.488+0000 7fabc7760640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:34:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:34:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:23.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:23.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:23 smithi102 ceph-mon[13158]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:34:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:34:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:34:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:34:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:24 smithi102 ceph-mon[13158]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:34:24.900 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.900 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.900 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.900 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.900 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.901 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:34:24.902 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.903 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.904 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[12201]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.905 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.906 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:34:24.907 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:34:24.908 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:34:24.909 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.910 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:24.911 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:24 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:34:26.505 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:26 smithi102 ceph-mon[13158]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:34:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:26 smithi070 ceph-mon[12201]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:34:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:26 smithi070 ceph-mon[17138]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:34:27.110 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:26 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:34:27.110 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 bash[49520]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:34:27.398 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.109Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:34:27.398 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:34:27.398 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:34:27.398 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[42045]: ts=2024-08-31T02:34:27.110Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:34:27.399 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 podman[49532]: 2024-08-31 02:34:27.120127527 +0000 UTC m=+0.104085539 container died c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:27.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 podman[49532]: 2024-08-31 02:34:27.476328111 +0000 UTC m=+0.460286120 container cleanup c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:27.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 bash[49532]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:34:27.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 bash[49559]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:34:28.093 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 podman[49546]: 2024-08-31 02:34:27.77760786 +0000 UTC m=+0.655080585 container remove c7c13a49447daa6545487cdc1a4f39a326fd72e990090e5fd2f87a27648bc4d9 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.781s CPU time. 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:27 smithi102 podman[49621]: 2024-08-31 02:34:27.99499728 +0000 UTC m=+0.108196987 container create 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:28.094 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 podman[49621]: 2024-08-31 02:34:27.903456413 +0000 UTC m=+0.016656129 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:34:28.417 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-mon[13158]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 podman[49621]: 2024-08-31 02:34:28.214613193 +0000 UTC m=+0.327812912 container init 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 podman[49621]: 2024-08-31 02:34:28.217614905 +0000 UTC m=+0.330814618 container start 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:34:28.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.241Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.242Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.243Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.244Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.247Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.247Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.734µs 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.247Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.248Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 bash[49621]: 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 2024-08-31T02:34:28.419 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:34:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-mon[12201]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:34:28.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-mon[17138]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE Bus STOPPING 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE Bus STOPPED 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE Bus STARTING 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE Serving on http://:::9283 2024-08-31T02:34:28.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:34:28] ENGINE Bus STARTED 2024-08-31T02:34:28.954 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.688Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-31T02:34:28.954 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.927Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.927Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.927Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.784µs wal_replay_duration=680.098924ms total_replay_duration=680.145128ms 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.930Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.931Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.931Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.954Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.339344ms db_storage=685ns remote_storage=1.222µs web_handler=367ns query_engine=645ns scrape=566.909µs scrape_sd=101.224µs notify=22.532µs notify_sd=9.385µs rules=22.255684ms 2024-08-31T02:34:28.955 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:34:28 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:34:28.954Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:29 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:34:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:34:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:29 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:34:30.378 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:30 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:30.378 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:30 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:30 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:32.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:32.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:32.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:31 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1492259492' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:34:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1492259492' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:34:32.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:32.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:31 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1492259492' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:34:34.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:33 smithi102 ceph-mon[13158]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:34.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:33 smithi070 ceph-mon[12201]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:34.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:33 smithi070 ceph-mon[17138]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:34:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:35] "GET /metrics HTTP/1.1" 200 37907 "" "Prometheus/2.33.4" 2024-08-31T02:34:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:36 smithi102 ceph-mon[13158]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-08-31T02:34:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:36 smithi070 ceph-mon[12201]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-08-31T02:34:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:36 smithi070 ceph-mon[17138]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-08-31T02:34:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:38 smithi102 ceph-mon[13158]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:34:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:38 smithi070 ceph-mon[12201]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:34:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:38 smithi070 ceph-mon[17138]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:34:40.565 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[12201]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[17138]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.566 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:40 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.631 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:40 smithi102 ceph-mon[13158]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:40.631 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:40 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.631 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:40 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:40.631 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:40 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:42.366 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:42 smithi070 ceph-mon[12201]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:42.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:42 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:42.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:42 smithi070 ceph-mon[17138]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:42.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:42 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:42 smithi102 ceph-mon[13158]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:34:42.617 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:42 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.362 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.600 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:44.601 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:45.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:45.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:34:45.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:34:45.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:45] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-08-31T02:34:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:46 smithi102 ceph-mon[13158]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:46 smithi070 ceph-mon[12201]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:46 smithi070 ceph-mon[17138]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:48.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:48 smithi102 ceph-mon[13158]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:48 smithi070 ceph-mon[12201]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:48.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:48 smithi070 ceph-mon[17138]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:50 smithi102 ceph-mon[13158]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:50 smithi070 ceph-mon[12201]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:50 smithi070 ceph-mon[17138]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:52 smithi102 ceph-mon[13158]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:52 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:52 smithi070 ceph-mon[12201]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:52 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:52 smithi070 ceph-mon[17138]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:52 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:34:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:54 smithi102 ceph-mon[13158]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:54 smithi070 ceph-mon[12201]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:54 smithi070 ceph-mon[17138]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:55.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:34:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:34:55] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-08-31T02:34:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:56 smithi102 ceph-mon[13158]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:56 smithi070 ceph-mon[12201]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:56 smithi070 ceph-mon[17138]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:34:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:34:58 smithi102 ceph-mon[13158]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:34:58 smithi070 ceph-mon[12201]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:34:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:34:58 smithi070 ceph-mon[17138]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:00 smithi102 ceph-mon[13158]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:00 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:00 smithi070 ceph-mon[12201]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:00 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:00 smithi070 ceph-mon[17138]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:00 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:02 smithi102 ceph-mon[13158]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:02 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:02 smithi070 ceph-mon[12201]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:02 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:02 smithi070 ceph-mon[17138]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:02 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.548 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.548 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:04.548 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:05.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:05] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-08-31T02:35:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:06 smithi102 ceph-mon[13158]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:06 smithi070 ceph-mon[12201]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:06 smithi070 ceph-mon[17138]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:08 smithi102 ceph-mon[13158]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:08 smithi070 ceph-mon[12201]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:08 smithi070 ceph-mon[17138]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:10 smithi102 ceph-mon[13158]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:10.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:10 smithi070 ceph-mon[12201]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:10.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:10 smithi070 ceph-mon[17138]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:12 smithi102 ceph-mon[13158]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:12 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:12 smithi070 ceph-mon[12201]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:12 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:12 smithi070 ceph-mon[17138]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:12 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:14 smithi102 ceph-mon[13158]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:14 smithi070 ceph-mon[12201]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:14 smithi070 ceph-mon[17138]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:15.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:15] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:35:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:16 smithi102 ceph-mon[13158]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:16 smithi070 ceph-mon[12201]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:16 smithi070 ceph-mon[17138]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:18 smithi102 ceph-mon[13158]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:18 smithi070 ceph-mon[12201]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:18 smithi070 ceph-mon[17138]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:20 smithi102 ceph-mon[13158]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:20.676 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:20 smithi070 ceph-mon[12201]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:20.677 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:20 smithi070 ceph-mon[17138]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:22.650 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:22 smithi102 ceph-mon[13158]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:22.650 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:22 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:22 smithi070 ceph-mon[12201]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:22 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:22.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:22 smithi070 ceph-mon[17138]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:22.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:22 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:24.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:24.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:24 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:24 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:25.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:25] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:35:26.570 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:26 smithi102 ceph-mon[13158]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:26 smithi070 ceph-mon[12201]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:26 smithi070 ceph-mon[17138]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:27 smithi102 ceph-mon[13158]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:27 smithi070 ceph-mon[12201]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:27 smithi070 ceph-mon[17138]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:29.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:29 smithi102 ceph-mon[13158]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:29.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:29 smithi070 ceph-mon[12201]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:29.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:29 smithi070 ceph-mon[17138]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:30.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:30 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:30.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:30 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:30.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:30 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:35:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:31 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:31.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:35:31.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:35:31.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:31 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:33 smithi102 ceph-mon[13158]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:33.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:33 smithi070 ceph-mon[12201]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:33.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:33 smithi070 ceph-mon[17138]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:35] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-08-31T02:35:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:36 smithi102 ceph-mon[13158]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:36 smithi070 ceph-mon[12201]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:36 smithi070 ceph-mon[17138]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:38 smithi070 ceph-mon[12201]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:38.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:38 smithi070 ceph-mon[17138]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:38 smithi102 ceph-mon[13158]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:40.600 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:40 smithi070 ceph-mon[12201]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:40.601 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:40 smithi070 ceph-mon[17138]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:40 smithi102 ceph-mon[13158]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:42.342 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:42 smithi070 ceph-mon[12201]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:42.342 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:42 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:42.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:42 smithi070 ceph-mon[17138]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:42.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:42 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:42.536 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:42 smithi102 ceph-mon[13158]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:42.537 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:42 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:35:45.398 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:45.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:45.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:35:45.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:45] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:35:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:46 smithi102 ceph-mon[13158]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:46 smithi070 ceph-mon[12201]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:46 smithi070 ceph-mon[17138]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:48.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:48 smithi102 ceph-mon[13158]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:48 smithi070 ceph-mon[12201]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:48.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:48 smithi070 ceph-mon[17138]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:50 smithi102 ceph-mon[13158]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:50 smithi070 ceph-mon[12201]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:50 smithi070 ceph-mon[17138]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:52 smithi102 ceph-mon[13158]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:52 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:52 smithi070 ceph-mon[12201]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:52 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:52 smithi070 ceph-mon[17138]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:52 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:35:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:54 smithi102 ceph-mon[13158]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:54 smithi070 ceph-mon[12201]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:54 smithi070 ceph-mon[17138]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:55.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:35:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:35:55] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:35:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:56 smithi102 ceph-mon[13158]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:56 smithi070 ceph-mon[12201]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:56 smithi070 ceph-mon[17138]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:35:58.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:35:58 smithi102 ceph-mon[13158]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:35:58 smithi070 ceph-mon[12201]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:35:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:35:58 smithi070 ceph-mon[17138]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:00 smithi102 ceph-mon[13158]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:00 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:00 smithi070 ceph-mon[12201]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:00 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:00 smithi070 ceph-mon[17138]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:00 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:02.643 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:02 smithi070 ceph-mon[12201]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:02.643 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:02 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:02.643 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:02 smithi070 ceph-mon[17138]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:02.643 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:02 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:02 smithi102 ceph-mon[13158]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:02 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.602 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:04 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:04.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:04 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:05.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:05] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-08-31T02:36:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:06 smithi102 ceph-mon[13158]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:06 smithi070 ceph-mon[12201]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:06 smithi070 ceph-mon[17138]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:08 smithi102 ceph-mon[13158]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:08 smithi070 ceph-mon[12201]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:08 smithi070 ceph-mon[17138]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:10 smithi102 ceph-mon[13158]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:10 smithi070 ceph-mon[12201]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:10 smithi070 ceph-mon[17138]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:12 smithi102 ceph-mon[13158]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:12 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:12 smithi070 ceph-mon[17138]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:12 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:12 smithi070 ceph-mon[12201]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:12 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:14.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:14 smithi102 ceph-mon[13158]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:14 smithi070 ceph-mon[12201]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:14 smithi070 ceph-mon[17138]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:15.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:15] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:36:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:16 smithi102 ceph-mon[13158]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:16 smithi070 ceph-mon[12201]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:16 smithi070 ceph-mon[17138]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:18 smithi102 ceph-mon[13158]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:18 smithi070 ceph-mon[12201]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:18 smithi070 ceph-mon[17138]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:20 smithi102 ceph-mon[13158]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:20 smithi070 ceph-mon[12201]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:20 smithi070 ceph-mon[17138]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:22.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:22 smithi070 ceph-mon[12201]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:22.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:22 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:22.663 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:22 smithi070 ceph-mon[17138]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:22.663 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:22 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:22 smithi102 ceph-mon[13158]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:22 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:23.588 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:23 smithi070 ceph-mon[12201]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:23.588 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:23 smithi070 ceph-mon[17138]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:23 smithi102 ceph-mon[13158]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.303 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.304 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:25.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:25] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-08-31T02:36:26.543 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:26 smithi102 ceph-mon[13158]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:26 smithi070 ceph-mon[12201]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:26 smithi070 ceph-mon[17138]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:28 smithi102 ceph-mon[13158]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:28 smithi070 ceph-mon[12201]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:28.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:28 smithi070 ceph-mon[17138]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:30 smithi102 ceph-mon[13158]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:30 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:30 smithi070 ceph-mon[12201]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:30 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:30 smithi070 ceph-mon[17138]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:30 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:36:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:36:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:36:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:36:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:34 smithi102 ceph-mon[13158]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:34 smithi070 ceph-mon[12201]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:34 smithi070 ceph-mon[17138]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:35] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-08-31T02:36:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:36 smithi102 ceph-mon[13158]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:36 smithi070 ceph-mon[12201]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:36 smithi070 ceph-mon[17138]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:38 smithi102 ceph-mon[13158]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:38 smithi070 ceph-mon[12201]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:38 smithi070 ceph-mon[17138]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:40 smithi102 ceph-mon[13158]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:40 smithi070 ceph-mon[12201]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:40 smithi070 ceph-mon[17138]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:42 smithi102 ceph-mon[13158]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:42 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:42 smithi070 ceph-mon[12201]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:42 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:42 smithi070 ceph-mon[17138]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:42 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:44.647 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:44 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:44 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:36:45.559 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:45 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:45.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:45 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:45.572 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:45 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:36:45.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:45] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-08-31T02:36:46.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:46 smithi102 ceph-mon[13158]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:46 smithi070 ceph-mon[12201]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:46 smithi070 ceph-mon[17138]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:48.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:48 smithi102 ceph-mon[13158]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:48 smithi070 ceph-mon[12201]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:48.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:48 smithi070 ceph-mon[17138]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:50 smithi102 ceph-mon[13158]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:50 smithi070 ceph-mon[12201]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:50 smithi070 ceph-mon[17138]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:52 smithi102 ceph-mon[13158]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:52 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:52 smithi070 ceph-mon[12201]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:52 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:52 smithi070 ceph-mon[17138]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:52.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:52 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:36:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:54 smithi102 ceph-mon[13158]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:54 smithi070 ceph-mon[12201]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:54 smithi070 ceph-mon[17138]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:55.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:36:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:36:55] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-08-31T02:36:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:56 smithi102 ceph-mon[13158]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:56 smithi070 ceph-mon[12201]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:56.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:56 smithi070 ceph-mon[17138]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:36:58.633 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:36:58 smithi102 ceph-mon[13158]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:36:58 smithi070 ceph-mon[12201]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:36:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:36:58 smithi070 ceph-mon[17138]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:00 smithi102 ceph-mon[13158]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:00 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:00 smithi070 ceph-mon[12201]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:00 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:00 smithi070 ceph-mon[17138]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:00 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:02 smithi102 ceph-mon[13158]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:02 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:02 smithi070 ceph-mon[12201]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:02 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:02 smithi070 ceph-mon[17138]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:02 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:04.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:04 smithi070 ceph-mon[17138]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:04.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:04 smithi070 ceph-mon[12201]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:04 smithi102 ceph-mon[13158]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:05.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[12201]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-mon[17138]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:05 smithi102 ceph-mon[13158]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:05.766 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:37:05] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-08-31T02:37:07.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:07 smithi102 ceph-mon[13158]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:07 smithi070 ceph-mon[12201]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:07.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:07 smithi070 ceph-mon[17138]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:09 smithi102 ceph-mon[13158]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:09.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:09 smithi070 ceph-mon[12201]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:09.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:09 smithi070 ceph-mon[17138]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:12 smithi102 ceph-mon[13158]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:12 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:12 smithi070 ceph-mon[12201]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:12 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:12 smithi070 ceph-mon[17138]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:12 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:14.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:14 smithi102 ceph-mon[13158]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:14 smithi070 ceph-mon[12201]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:14 smithi070 ceph-mon[17138]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:15 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:15.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:15 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:15 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:37:15] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-08-31T02:37:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:16 smithi102 ceph-mon[13158]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:16 smithi070 ceph-mon[12201]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:16 smithi070 ceph-mon[17138]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:18.644 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:18 smithi070 ceph-mon[12201]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:18.645 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:18 smithi070 ceph-mon[17138]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:18 smithi102 ceph-mon[13158]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:19.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:37:20.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:20 smithi070 ceph-mon[12201]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:20.587 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:20 smithi070 ceph-mon[17138]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:20 smithi102 ceph-mon[13158]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:21.345 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 16s ago 22m - - 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 16s ago 22m - - 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (9m) 16s ago 17m 24.1M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (8m) 16s ago 17m 54.0M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (2m) 16s ago 16m 53.0M - 3.5 e1d6a67b021e f09e8859b57d 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (11m) 16s ago 22m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (6m) 16s ago 23m 505M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (23m) 16s ago 23m 73.5M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (22m) 16s ago 22m 54.8M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (22m) 16s ago 22m 54.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:37:21.346 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (9m) 16s ago 18m 17.0M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (9m) 16s ago 18m 16.4M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (21m) 16s ago 21m 56.1M 2736M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (21m) 16s ago 21m 57.5M 2736M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (21m) 16s ago 21m 49.6M 2736M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (20m) 16s ago 20m 52.7M 2736M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (20m) 16s ago 20m 53.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (19m) 16s ago 19m 50.0M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (19m) 16s ago 19m 51.9M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (19m) 16s ago 19m 56.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 16s ago 18m 65.1M - 2.33.4 514e6a882f6e 668e0fa9edff 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (16m) 16s ago 16m 76.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:37:21.347 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (16m) 16s ago 16m 75.7M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:37:21.971 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions' 2024-08-31T02:37:22.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[12201]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:22.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:22.547 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[12201]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:37:22.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[17138]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:22.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:22.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:22 smithi070 ceph-mon[17138]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:37:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:22 smithi102 ceph-mon[13158]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:22 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:22 smithi102 ceph-mon[13158]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:37:24.114 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:37:24.115 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:37:24.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:24 smithi070 ceph-mon[12201]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:24.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:24 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1364419719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:24.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:24 smithi070 ceph-mon[17138]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:24.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:24 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1364419719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:24 smithi102 ceph-mon[13158]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:24 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1364419719' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:24.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph -s' 2024-08-31T02:37:25.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:37:25] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:25.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:25 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:25 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:27.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:27 smithi070 ceph-mon[12201]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:27.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:27 smithi070 ceph-mon[17138]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: cluster: 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: id: 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: health: HEALTH_OK 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: services: 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: mgr: y(active, since 3m), standbys: x 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: osd: 8 osds: 8 up (since 19m), 8 in (since 19m) 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: data: 2024-08-31T02:37:27.843 INFO:teuthology.orchestra.run.smithi070.stdout: pools: 6 pools, 161 pgs 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: objects: 241 objects, 585 KiB 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: usage: 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: pgs: 161 active+clean 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: io: 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-08-31T02:37:27.844 INFO:teuthology.orchestra.run.smithi070.stdout: 2024-08-31T02:37:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:27 smithi102 ceph-mon[13158]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:28.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:28 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/645048044' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:37:28.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:28 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/645048044' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:37:28.654 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:28 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/645048044' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-31T02:37:28.678 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:37:29.748 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:29 smithi070 ceph-mon[12201]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:29.749 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:29 smithi070 ceph-mon[17138]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:29.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:29 smithi102 ceph-mon[13158]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:30.645 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:30 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:30.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:30 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:30.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:30 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:31.147 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_OK 2024-08-31T02:37:31.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[12201]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:31.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1035772636' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:37:31.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:31.867 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[17138]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:31.867 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1035772636' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:37:31.867 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:31 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:31 smithi102 ceph-mon[13158]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:37:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:31 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1035772636' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:37:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:31 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:32.033 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[12201]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:32.572 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:32.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:32.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:32 smithi070 ceph-mon[17138]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:32 smithi102 ceph-mon[13158]: from='mgr.25063 172.21.15.70:0/733343268' entity='mgr.y' 2024-08-31T02:37:34.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:33 smithi102 ceph-mon[13158]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:34.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:33 smithi070 ceph-mon[12201]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:34.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:33 smithi070 ceph-mon[17138]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:34.240 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:37:34.866 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:34 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3676666919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:34.867 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:34 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3676666919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:34.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph mgr fail' 2024-08-31T02:37:35.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:34 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3676666919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:37:35.848 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:37:35] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-08-31T02:37:36.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:36 smithi070 ceph-mon[12201]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:36.374 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:36 smithi070 ceph-mon[17138]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:36 smithi102 ceph-mon[13158]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:37:37.596 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:37:37.596 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-mon[13158]: osdmap e106: 8 total, 8 up, 8 in 2024-08-31T02:37:37.596 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE Bus STOPPING 2024-08-31T02:37:37.596 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:37:37.596 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE Bus STOPPED 2024-08-31T02:37:37.599 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-mon[12201]: osdmap e106: 8 total, 8 up, 8 in 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setuser ceph since I am not root 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setgroup ceph since I am not root 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-mgr[56919]: -- 172.21.15.70:0/3655149874 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5594402930e0 con 0x559440271000 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:37.473+0000 7f8d10a2d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:37.535+0000 7f8d10a2d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-08-31T02:37:37.600 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-mon[17138]: osdmap e106: 8 total, 8 up, 8 in 2024-08-31T02:37:37.914 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:37.694+0000 7f8d10a2d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:37:37.914 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:37.765+0000 7f8d10a2d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:37:37.917 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE Bus STARTING 2024-08-31T02:37:37.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE Serving on http://:::9283 2024-08-31T02:37:37.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:37 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:37] ENGINE Bus STARTED 2024-08-31T02:37:37.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'sleep 180' 2024-08-31T02:37:38.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:37.925+0000 7f8d10a2d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:37:38.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:38.076+0000 7f8d10a2d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:37:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: mgrmap e33: x(active, starting, since 0.44777s) 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:37:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: Manager daemon x is now available 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.509 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:38.418+0000 7f8d10a2d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:37:38.509 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:38.483+0000 7f8d10a2d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:37:38.509 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:37:38.509 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: mgrmap e33: x(active, starting, since 0.44777s) 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:37:38.510 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: Manager daemon x is now available 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.511 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3969493687' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: mgrmap e33: x(active, starting, since 0.44777s) 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:37:38.512 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: Manager daemon x is now available 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:38.513 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.514 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:38.867 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:38 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:38.679+0000 7f8d10a2d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:37:39.103 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:37:38.923+0000 7fd08ef5f640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:37:38] ENGINE Error in HTTPServer.serve 2024-08-31T02:37:39.103 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Traceback (most recent call last): 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._connections.run(self.expiration_interval) 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._run(expiration_interval) 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: s = self.context.wrap_socket( 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return self.sslsocket_class._create( 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:37:39.104 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self.do_handshake() 2024-08-31T02:37:39.105 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:37:39.105 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self._sslobj.do_handshake() 2024-08-31T02:37:39.105 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:37:39.105 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:37:39.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: mgrmap e34: x(active, since 1.4522s) 2024-08-31T02:37:39.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:39.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:39.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.408 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.408 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:37:39.408 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:39.408 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:39 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: mgrmap e34: x(active, since 1.4522s) 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:39.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: mgrmap e34: x(active, since 1.4522s) 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]': finished 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:39.611 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:39.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:39.639+0000 7f8d10a2d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:37:39.868 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:39.714+0000 7f8d10a2d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:37:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: [31/Aug/2024:02:37:38] ENGINE Bus STARTING 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: [31/Aug/2024:02:37:38] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: [31/Aug/2024:02:37:38] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: [31/Aug/2024:02:37:38] ENGINE Bus STARTED 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: [31/Aug/2024:02:37:38] ENGINE Error in HTTPServer.serve 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:37:40.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:37:40.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:40.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:40 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:40.520+0000 7f8d10a2d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:37:40.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:40.637+0000 7f8d10a2d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:37:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: [31/Aug/2024:02:37:38] ENGINE Bus STARTING 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: [31/Aug/2024:02:37:38] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: [31/Aug/2024:02:37:38] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: [31/Aug/2024:02:37:38] ENGINE Bus STARTED 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: [31/Aug/2024:02:37:38] ENGINE Error in HTTPServer.serve 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:37:40.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: [31/Aug/2024:02:37:38] ENGINE Bus STARTING 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: [31/Aug/2024:02:37:38] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: [31/Aug/2024:02:37:38] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: [31/Aug/2024:02:37:38] ENGINE Bus STARTED 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: [31/Aug/2024:02:37:38] ENGINE Error in HTTPServer.serve 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:37:40.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:40.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:40.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.077 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:40.790+0000 7f8d10a2d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:37:41.077 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:40.853+0000 7f8d10a2d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:37:41.077 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.008+0000 7f8d10a2d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:37:41.077 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.076+0000 7f8d10a2d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:37:41.330 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.139+0000 7f8d10a2d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: mgrmap e35: x(active, since 3s) 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:41 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.638 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.329+0000 7f8d10a2d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:37:41.638 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.419+0000 7f8d10a2d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:37:41.638 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.491+0000 7f8d10a2d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:37:41.638 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.578+0000 7f8d10a2d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: mgrmap e35: x(active, since 3s) 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: mgrmap e35: x(active, since 3s) 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.640 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:41.944 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.639+0000 7f8d10a2d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:37:41.944 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:41.700+0000 7f8d10a2d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:37:42.759 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:42.759 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:42.759 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.759 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.759 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.760 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.760 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.760 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.760 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:42 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:42.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:42.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.931 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.931 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.931 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.931 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.932 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:42.534+0000 7f8d10a2d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:37:42.932 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:37:42] ENGINE Bus STARTING 2024-08-31T02:37:42.932 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: CherryPy Checker: 2024-08-31T02:37:42.932 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: The Application mounted at '' has an empty config. 2024-08-31T02:37:42.933 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:37:42.933 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:37:42] ENGINE Serving on http://:::9283 2024-08-31T02:37:42.933 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:37:42] ENGINE Bus STARTED 2024-08-31T02:37:42.933 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='client.15231 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.934 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.935 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:42.935 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:42 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[12201]: Standby manager daemon y started 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:37:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[17138]: Standby manager daemon y started 2024-08-31T02:37:43.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:37:43.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:37:43.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:37:43.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:43 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:37:43.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:43 smithi102 ceph-mon[13158]: Standby manager daemon y started 2024-08-31T02:37:43.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:37:43.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:37:43.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:37:43.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:43 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:37:44.243+0000 7fd0c89dd640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Creating ceph-iscsi config... 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:44.309 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Traceback (most recent call last): 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return _run_code(code, main_globals, None, 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: exec(code, run_globals) 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:44.310 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Traceback (most recent call last): 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return self.wait_async( 2024-08-31T02:37:44.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return future.result(timeout) 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return self.__get_result() 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: raise self._exception 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: out, err, code = await self._run_cephadm( 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: raise OrchestratorError( 2024-08-31T02:37:44.312 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Creating ceph-iscsi config... 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: Traceback (most recent call last): 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: return _run_code(code, main_globals, None, 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: exec(code, run_globals) 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:44.313 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:44.314 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: mgrmap e36: x(active, since 6s), standbys: y 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:44 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: mgrmap e36: x(active, since 6s), standbys: y 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.672 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: mgrmap e36: x(active, since 6s), standbys: y 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:44.673 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:44 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:45.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:37:45.576 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.577 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[12201]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[12201]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.579 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.580 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:37:45.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:37:45.583 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:37:45.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:45 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.766 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.767 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:37:45.768 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:37:45.769 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:37:45.770 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:45 smithi102 ceph-mon[13158]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:37:45.771 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:45 smithi102 ceph-mon[13158]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:37:45.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:37:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:37:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-08-31T02:37:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:46 smithi102 ceph-mon[13158]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:37:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[12201]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:37:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[17138]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:37:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:47.755 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:37:47.755 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 bash[54114]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:37:48.032 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:37:48.033 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[49635]: ts=2024-08-31T02:37:47.755Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:37:48.033 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:47 smithi102 podman[54125]: 2024-08-31 02:37:47.765317187 +0000 UTC m=+0.135310917 container died 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.389 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54125]: 2024-08-31 02:37:48.120874106 +0000 UTC m=+0.490867840 container cleanup 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.389 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 bash[54125]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:37:48.389 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 bash[54150]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:37:48.648 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-mon[13158]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54137]: 2024-08-31 02:37:48.388824049 +0000 UTC m=+0.625022399 container remove 668e0fa9edffefaf95c5271c2e600db99234e7c9dc3a2cd0dc5e252a07a050a8 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.301s CPU time. 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54217]: 2024-08-31 02:37:48.525545894 +0000 UTC m=+0.017023601 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:37:48.648 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54217]: 2024-08-31 02:37:48.6481651 +0000 UTC m=+0.139642806 container create 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:48 smithi070 ceph-mon[12201]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:37:48.857 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:48 smithi070 ceph-mon[17138]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54217]: 2024-08-31 02:37:48.852950962 +0000 UTC m=+0.344428669 container init 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 podman[54217]: 2024-08-31 02:37:48.855846308 +0000 UTC m=+0.347324025 container start 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:37:48.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.879Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.881Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.881Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.883Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.884Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.884Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.535µs 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.884Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:37:48.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:48.885Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-08-31T02:37:49.317 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 bash[54217]: 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 2024-08-31T02:37:49.317 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:48 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE Bus STOPPING 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE Bus STOPPED 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE Bus STARTING 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE Serving on http://:::9283 2024-08-31T02:37:49.317 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:37:49] ENGINE Bus STARTED 2024-08-31T02:37:49.599 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.357Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.599Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.601Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.601Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.601Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=51.48µs wal_replay_duration=717.719746ms total_replay_duration=717.783363ms 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.606Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.606Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.606Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:37:49.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.629Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.702228ms db_storage=759ns remote_storage=1.298µs web_handler=451ns query_engine=652ns scrape=209.519µs scrape_sd=97.162µs notify=16.384µs notify_sd=14.977µs rules=21.968557ms 2024-08-31T02:37:49.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:37:49.629Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:37:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[12201]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:50.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:49 smithi070 ceph-mon[17138]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:37:50.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:50.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:37:50.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:49 smithi102 ceph-mon[13158]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/157228294' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:52 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/157228294' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:37:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:37:52.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:37:52.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:37:52.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:52 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/157228294' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:37:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:53 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:53 smithi102 ceph-mon[13158]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:53 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2204204485' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:53 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:53 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[12201]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2204204485' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-08-31T02:37:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[17138]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-08-31T02:37:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2204204485' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]: dispatch 2024-08-31T02:37:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:53 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:37:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:54 smithi102 ceph-mon[13158]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:54 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]': finished 2024-08-31T02:37:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:54 smithi102 ceph-mon[13158]: osdmap e107: 8 total, 8 up, 8 in 2024-08-31T02:37:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:54 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1332358707' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:54 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[12201]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]': finished 2024-08-31T02:37:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[12201]: osdmap e107: 8 total, 8 up, 8 in 2024-08-31T02:37:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1332358707' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[17138]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:37:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2362188023"}]': finished 2024-08-31T02:37:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[17138]: osdmap e107: 8 total, 8 up, 8 in 2024-08-31T02:37:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1332358707' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:54 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]: dispatch 2024-08-31T02:37:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:55 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]': finished 2024-08-31T02:37:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:55 smithi102 ceph-mon[13158]: osdmap e108: 8 total, 8 up, 8 in 2024-08-31T02:37:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:55 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1067041409' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:55 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]': finished 2024-08-31T02:37:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[12201]: osdmap e108: 8 total, 8 up, 8 in 2024-08-31T02:37:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1067041409' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6801/1015789811"}]': finished 2024-08-31T02:37:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[17138]: osdmap e108: 8 total, 8 up, 8 in 2024-08-31T02:37:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1067041409' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:55 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]: dispatch 2024-08-31T02:37:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:56 smithi102 ceph-mon[13158]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:37:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:56 smithi102 ceph-mon[13158]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]': finished 2024-08-31T02:37:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:56 smithi102 ceph-mon[13158]: osdmap e109: 8 total, 8 up, 8 in 2024-08-31T02:37:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:56 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]: dispatch 2024-08-31T02:37:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[17138]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:37:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[17138]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]': finished 2024-08-31T02:37:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[17138]: osdmap e109: 8 total, 8 up, 8 in 2024-08-31T02:37:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]: dispatch 2024-08-31T02:37:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[12201]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:37:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[12201]: from='client.? ' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:6800/1015789811"}]': finished 2024-08-31T02:37:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[12201]: osdmap e109: 8 total, 8 up, 8 in 2024-08-31T02:37:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:56 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]: dispatch 2024-08-31T02:37:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:57 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]': finished 2024-08-31T02:37:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:57 smithi102 ceph-mon[13158]: osdmap e110: 8 total, 8 up, 8 in 2024-08-31T02:37:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:57 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]: dispatch 2024-08-31T02:37:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]': finished 2024-08-31T02:37:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[12201]: osdmap e110: 8 total, 8 up, 8 in 2024-08-31T02:37:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]: dispatch 2024-08-31T02:37:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3873578522' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/1746253841"}]': finished 2024-08-31T02:37:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[17138]: osdmap e110: 8 total, 8 up, 8 in 2024-08-31T02:37:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:57 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]: dispatch 2024-08-31T02:37:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:58 smithi102 ceph-mon[13158]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:58 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]': finished 2024-08-31T02:37:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:58 smithi102 ceph-mon[13158]: osdmap e111: 8 total, 8 up, 8 in 2024-08-31T02:37:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:58 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]: dispatch 2024-08-31T02:37:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[12201]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]': finished 2024-08-31T02:37:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[12201]: osdmap e111: 8 total, 8 up, 8 in 2024-08-31T02:37:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]: dispatch 2024-08-31T02:37:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[17138]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:37:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2270071360' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2303740194"}]': finished 2024-08-31T02:37:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[17138]: osdmap e111: 8 total, 8 up, 8 in 2024-08-31T02:37:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:58 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]: dispatch 2024-08-31T02:37:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:59 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]': finished 2024-08-31T02:37:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:37:59 smithi102 ceph-mon[13158]: osdmap e112: 8 total, 8 up, 8 in 2024-08-31T02:37:59.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:59 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]': finished 2024-08-31T02:37:59.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:37:59 smithi070 ceph-mon[12201]: osdmap e112: 8 total, 8 up, 8 in 2024-08-31T02:37:59.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:59 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/699149729' entity='client.iscsi.foo.smithi070.ashysy' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.70:0/2603903140"}]': finished 2024-08-31T02:37:59.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:37:59 smithi070 ceph-mon[17138]: osdmap e112: 8 total, 8 up, 8 in 2024-08-31T02:38:00.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:00 smithi102 ceph-mon[13158]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:00 smithi070 ceph-mon[12201]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:00 smithi070 ceph-mon[17138]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:02 smithi070 ceph-mon[12201]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-08-31T02:38:02.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:02 smithi070 ceph-mon[17138]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-08-31T02:38:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:02 smithi102 ceph-mon[13158]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 97 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-08-31T02:38:03.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:03 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:03 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:03.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:03 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:03.917 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:03] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-08-31T02:38:04.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:04 smithi070 ceph-mon[12201]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:38:04.687 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:04 smithi070 ceph-mon[17138]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:38:04.727 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:04 smithi102 ceph-mon[13158]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:38:05.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:05 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:05 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:05 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.812 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:05 smithi102 ceph-mon[13158]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:05.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[12201]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:05.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:05 smithi070 ceph-mon[17138]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:08 smithi102 ceph-mon[13158]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:38:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:08 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:08 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:08 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:08 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[12201]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[17138]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:08.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:08 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:10 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:10.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:10 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:12 smithi102 ceph-mon[13158]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:12 smithi070 ceph-mon[12201]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:12 smithi070 ceph-mon[17138]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:13 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:13.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:13 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:13 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:13] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-08-31T02:38:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:14 smithi102 ceph-mon[13158]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:14 smithi070 ceph-mon[12201]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:14 smithi070 ceph-mon[17138]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:16 smithi102 ceph-mon[13158]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:16 smithi070 ceph-mon[12201]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:16 smithi070 ceph-mon[17138]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:18 smithi102 ceph-mon[13158]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:18 smithi070 ceph-mon[12201]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:18 smithi070 ceph-mon[17138]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:20 smithi102 ceph-mon[13158]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:20 smithi070 ceph-mon[12201]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:20 smithi070 ceph-mon[17138]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:22 smithi102 ceph-mon[13158]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:22 smithi070 ceph-mon[12201]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:22 smithi070 ceph-mon[17138]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:23.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:23 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:23.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:23 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:23 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:23 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:23.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:23 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:23.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:23 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:23.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:23] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-08-31T02:38:24.660 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:24 smithi070 ceph-mon[12201]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:24.660 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:24 smithi070 ceph-mon[17138]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:24.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:24 smithi102 ceph-mon[13158]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:26.329 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:26 smithi102 ceph-mon[13158]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:26.330 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:26 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.330 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:26 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.330 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:26 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.474 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[12201]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:26.474 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[17138]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:26.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:26 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:27 smithi102 ceph-mon[13158]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:28.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[12201]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:28.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:27 smithi070 ceph-mon[17138]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:30 smithi102 ceph-mon[13158]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:30 smithi070 ceph-mon[12201]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:30 smithi070 ceph-mon[17138]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:32 smithi102 ceph-mon[13158]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:32 smithi070 ceph-mon[12201]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:32 smithi070 ceph-mon[17138]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:33 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:33 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:33 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:33] "GET /metrics HTTP/1.1" 200 37905 "" "Prometheus/2.33.4" 2024-08-31T02:38:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:34 smithi102 ceph-mon[13158]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:34 smithi070 ceph-mon[12201]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:34 smithi070 ceph-mon[17138]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:36 smithi102 ceph-mon[13158]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:36 smithi070 ceph-mon[12201]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:36 smithi070 ceph-mon[17138]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:38 smithi102 ceph-mon[13158]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:38 smithi070 ceph-mon[12201]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:38.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:38 smithi070 ceph-mon[17138]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:40 smithi102 ceph-mon[13158]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:40 smithi070 ceph-mon[12201]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:40 smithi070 ceph-mon[17138]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:42 smithi102 ceph-mon[13158]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:42 smithi070 ceph-mon[12201]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:42 smithi070 ceph-mon[17138]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:43.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:43 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:43 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:43 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:43.874 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:43] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-08-31T02:38:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:44 smithi102 ceph-mon[13158]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:44.694 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:44 smithi070 ceph-mon[12201]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:44.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:44 smithi070 ceph-mon[17138]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[12201]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[17138]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:46.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.580 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:46 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:46 smithi102 ceph-mon[13158]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:46 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:47 smithi102 ceph-mon[13158]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[12201]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:48.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:47 smithi070 ceph-mon[17138]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:50 smithi102 ceph-mon[13158]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:50 smithi070 ceph-mon[12201]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:50 smithi070 ceph-mon[17138]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:52 smithi102 ceph-mon[13158]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:38:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:38:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:38:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[12201]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:38:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[17138]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:38:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:38:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:53 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:53 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:53 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:53 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:53.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:53 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:38:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:53 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:38:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:38:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:38:53] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-08-31T02:38:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:54 smithi102 ceph-mon[13158]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:54 smithi070 ceph-mon[12201]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:54.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:54 smithi070 ceph-mon[17138]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:38:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:56 smithi102 ceph-mon[13158]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:56 smithi070 ceph-mon[12201]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:56 smithi070 ceph-mon[17138]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:38:58 smithi102 ceph-mon[13158]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:38:58 smithi070 ceph-mon[12201]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:38:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:38:58 smithi070 ceph-mon[17138]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:00 smithi102 ceph-mon[13158]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:00 smithi070 ceph-mon[12201]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:00 smithi070 ceph-mon[17138]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:02 smithi102 ceph-mon[13158]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:02 smithi070 ceph-mon[12201]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:02.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:02 smithi070 ceph-mon[17138]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:03 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:03 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:03 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:04.042 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:03] "GET /metrics HTTP/1.1" 200 37902 "" "Prometheus/2.33.4" 2024-08-31T02:39:04.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:04 smithi070 ceph-mon[12201]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:04.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:04 smithi070 ceph-mon[17138]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:04 smithi102 ceph-mon[13158]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:06.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[12201]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:06.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:06.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:06.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[17138]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:06.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:06.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:06 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:06 smithi102 ceph-mon[13158]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:06 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:06 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:08 smithi102 ceph-mon[13158]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:08 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:08 smithi070 ceph-mon[12201]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:08 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:08.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:08 smithi070 ceph-mon[17138]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:08 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:10 smithi102 ceph-mon[13158]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:10 smithi070 ceph-mon[12201]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:10 smithi070 ceph-mon[17138]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:12 smithi102 ceph-mon[13158]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:12 smithi070 ceph-mon[12201]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:12 smithi070 ceph-mon[17138]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:13 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:13 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:13 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:13] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-08-31T02:39:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:14 smithi102 ceph-mon[13158]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:14 smithi070 ceph-mon[12201]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:14 smithi070 ceph-mon[17138]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:16 smithi102 ceph-mon[13158]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:16 smithi070 ceph-mon[12201]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:16 smithi070 ceph-mon[17138]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:18 smithi102 ceph-mon[13158]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:18 smithi070 ceph-mon[12201]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:18 smithi070 ceph-mon[17138]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:20 smithi102 ceph-mon[13158]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:20 smithi070 ceph-mon[12201]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:20.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:20 smithi070 ceph-mon[17138]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:22 smithi102 ceph-mon[13158]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:22 smithi070 ceph-mon[12201]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:22 smithi070 ceph-mon[17138]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:23 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:23 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:23 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:23 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:23 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:23 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:23.983 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:23] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:39:24.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:24 smithi070 ceph-mon[12201]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:24.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:24 smithi070 ceph-mon[17138]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:24 smithi102 ceph-mon[13158]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:26.569 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:26 smithi070 ceph-mon[12201]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:26.569 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:26 smithi070 ceph-mon[17138]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:26 smithi102 ceph-mon[13158]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:28 smithi070 ceph-mon[12201]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:28 smithi070 ceph-mon[17138]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:28.820 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:28 smithi102 ceph-mon[13158]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:30 smithi102 ceph-mon[13158]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:30 smithi070 ceph-mon[12201]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:30 smithi070 ceph-mon[17138]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:32 smithi070 ceph-mon[12201]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:32 smithi070 ceph-mon[17138]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:32.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:32 smithi102 ceph-mon[13158]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:33 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:33 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:33 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:33] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:39:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:34 smithi070 ceph-mon[12201]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:34 smithi070 ceph-mon[17138]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:34 smithi102 ceph-mon[13158]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:36 smithi102 ceph-mon[13158]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:36 smithi070 ceph-mon[12201]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:36 smithi070 ceph-mon[17138]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:38 smithi102 ceph-mon[13158]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:38 smithi070 ceph-mon[12201]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:38 smithi070 ceph-mon[17138]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:40 smithi102 ceph-mon[13158]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:40 smithi070 ceph-mon[12201]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:40 smithi070 ceph-mon[17138]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:42 smithi102 ceph-mon[13158]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:42 smithi070 ceph-mon[12201]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:42.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:42 smithi070 ceph-mon[17138]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:43 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:43 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:43 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:43.961 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:43] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-08-31T02:39:44.592 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:44 smithi102 ceph-mon[13158]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:44 smithi070 ceph-mon[12201]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:44 smithi070 ceph-mon[17138]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:46.354 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:46 smithi102 ceph-mon[13158]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:46.586 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:46 smithi070 ceph-mon[12201]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:46.586 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:46 smithi070 ceph-mon[17138]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:47 smithi102 ceph-mon[13158]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[17138]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:47.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:47 smithi070 ceph-mon[12201]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:50 smithi102 ceph-mon[13158]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:50 smithi070 ceph-mon[12201]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:50 smithi070 ceph-mon[17138]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:52 smithi102 ceph-mon[13158]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:39:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:39:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:39:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[12201]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:39:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:39:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[17138]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:39:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:39:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:53 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:53 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:53 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:53 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:53 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:39:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:53 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:39:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:39:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:39:53] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-08-31T02:39:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:54 smithi102 ceph-mon[13158]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:54 smithi070 ceph-mon[12201]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:54 smithi070 ceph-mon[17138]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:39:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:56 smithi102 ceph-mon[13158]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:56 smithi070 ceph-mon[12201]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:56 smithi070 ceph-mon[17138]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:39:58 smithi102 ceph-mon[13158]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:39:58 smithi070 ceph-mon[12201]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:39:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:39:58 smithi070 ceph-mon[17138]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:00 smithi102 ceph-mon[13158]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:00 smithi102 ceph-mon[13158]: overall HEALTH_OK 2024-08-31T02:40:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:00 smithi070 ceph-mon[12201]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:00 smithi070 ceph-mon[12201]: overall HEALTH_OK 2024-08-31T02:40:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:00 smithi070 ceph-mon[17138]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:00 smithi070 ceph-mon[17138]: overall HEALTH_OK 2024-08-31T02:40:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:02 smithi102 ceph-mon[13158]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:02.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:02 smithi070 ceph-mon[17138]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:02 smithi070 ceph-mon[12201]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:03 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:03.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:03 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:03 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:03.932 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:03] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-08-31T02:40:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:04 smithi102 ceph-mon[13158]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:04.674 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:04 smithi070 ceph-mon[12201]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:04.674 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:04 smithi070 ceph-mon[17138]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:06 smithi102 ceph-mon[13158]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:06 smithi070 ceph-mon[12201]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:06.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:06 smithi070 ceph-mon[17138]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:07.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:07.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:07 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:07 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:10 smithi102 ceph-mon[13158]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:10 smithi070 ceph-mon[12201]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:10.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:10 smithi070 ceph-mon[17138]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:12 smithi102 ceph-mon[13158]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:12 smithi070 ceph-mon[17138]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:12 smithi070 ceph-mon[12201]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:13 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:13 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:13 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:13] "GET /metrics HTTP/1.1" 200 37911 "" "Prometheus/2.33.4" 2024-08-31T02:40:14.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:14 smithi102 ceph-mon[13158]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:14 smithi070 ceph-mon[17138]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:14 smithi070 ceph-mon[12201]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:16 smithi102 ceph-mon[13158]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:16 smithi070 ceph-mon[17138]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:16 smithi070 ceph-mon[12201]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:18.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:18 smithi102 ceph-mon[13158]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:18 smithi070 ceph-mon[12201]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:18.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:18 smithi070 ceph-mon[17138]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:20.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:20 smithi102 ceph-mon[13158]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:20.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:20 smithi070 ceph-mon[12201]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:20.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:20 smithi070 ceph-mon[17138]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:22.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:22 smithi102 ceph-mon[13158]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:22 smithi070 ceph-mon[12201]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:22.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:22 smithi070 ceph-mon[17138]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:23.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:23 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:23.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:23 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:23 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:23 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:23 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:23 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:24.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:23] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-08-31T02:40:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:24 smithi070 ceph-mon[12201]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:24 smithi070 ceph-mon[17138]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:24.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:24 smithi102 ceph-mon[13158]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:26.666 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:26 smithi102 ceph-mon[13158]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:26 smithi070 ceph-mon[12201]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:26 smithi070 ceph-mon[17138]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:28.107 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.108 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:28.109 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.109 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.109 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:27 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:28.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:27 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:30.518 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:30 smithi070 ceph-mon[12201]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:30.519 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:30 smithi070 ceph-mon[17138]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:30 smithi102 ceph-mon[13158]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:32 smithi102 ceph-mon[13158]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:32 smithi070 ceph-mon[12201]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:32 smithi070 ceph-mon[17138]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:33.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:33 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:33 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:33 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:33] "GET /metrics HTTP/1.1" 200 37910 "" "Prometheus/2.33.4" 2024-08-31T02:40:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:34 smithi070 ceph-mon[12201]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:34 smithi070 ceph-mon[17138]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:34 smithi102 ceph-mon[13158]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:36 smithi102 ceph-mon[13158]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:36 smithi070 ceph-mon[12201]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:36.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:36 smithi070 ceph-mon[17138]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:38 smithi102 ceph-mon[13158]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:38 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:38 smithi070 ceph-mon[12201]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:38 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:38 smithi070 ceph-mon[17138]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:38 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:40.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:40:40.693 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:40 smithi070 ceph-mon[12201]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:40.694 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:40 smithi070 ceph-mon[17138]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:40.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:40 smithi102 ceph-mon[13158]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:40:42.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:42 smithi070 ceph-mon[12201]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:42.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:42 smithi070 ceph-mon[17138]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:42.650 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:40:42.650 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 15s ago 26m - - 2024-08-31T02:40:42.650 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 15s ago 26m - - 2024-08-31T02:40:42.650 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (12m) 15s ago 21m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (12m) 15s ago 20m 54.0M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (2m) 15s ago 19m 51.0M - 3.5 e1d6a67b021e 3b20e62d1fc2 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (15m) 15s ago 25m 503M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (9m) 15s ago 27m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (27m) 15s ago 27m 77.8M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (25m) 15s ago 25m 64.1M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (25m) 15s ago 25m 61.1M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (12m) 15s ago 21m 17.0M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (12m) 15s ago 21m 16.7M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (25m) 15s ago 25m 56.8M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (24m) 15s ago 24m 57.7M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:40:42.651 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (24m) 15s ago 24m 50.1M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (24m) 15s ago 24m 53.1M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (23m) 15s ago 23m 54.3M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (23m) 15s ago 23m 50.4M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (22m) 15s ago 22m 52.7M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (22m) 15s ago 22m 56.5M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 15s ago 21m 70.0M - 2.33.4 514e6a882f6e 77434e695a34 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (20m) 15s ago 20m 77.6M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:40:42.652 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (19m) 15s ago 19m 76.5M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:40:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:42 smithi102 ceph-mon[13158]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:40:43.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ls' 2024-08-31T02:40:43.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:43 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:43 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:43 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:44.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:43] "GET /metrics HTTP/1.1" 200 37909 "" "Prometheus/2.33.4" 2024-08-31T02:40:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:44 smithi070 ceph-mon[12201]: from='client.25276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:44 smithi070 ceph-mon[12201]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:44.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:44 smithi070 ceph-mon[17138]: from='client.25276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:44 smithi070 ceph-mon[17138]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:44.917 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:44 smithi102 ceph-mon[13158]: from='client.25276 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:44 smithi102 ceph-mon[13158]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:45.509 INFO:teuthology.orchestra.run.smithi070.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:agent 2/2 18s ago 3m * 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager ?:9093,9094 1/1 18s ago 21m smithi070=a;count:1 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:grafana ?:3000 1/1 18s ago 21m smithi102=a;count:1 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo ?:5000 1/1 18s ago 19m count:1 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:mgr 2/2 18s ago 25m smithi070=y;smithi102=x;count:2 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:mon 3/3 18s ago 26m smithi070:172.21.15.70=a;smithi070:[v2:172.21.15.70:3301,v1:172.21.15.70:6790]=c;smithi102:172.21.15.102=b;count:3 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter ?:9100 2/2 18s ago 22m smithi070=a;smithi102=b;count:2 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:osd 8 18s ago - 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus ?:9095 1/1 18s ago 22m smithi102=a;count:1 2024-08-31T02:40:45.510 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo ?:8000 2/2 18s ago 20m count:2 2024-08-31T02:40:46.150 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions' 2024-08-31T02:40:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:46 smithi070 ceph-mon[12201]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:46 smithi070 ceph-mon[12201]: from='client.15375 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:46.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:46 smithi070 ceph-mon[17138]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:46.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:46 smithi070 ceph-mon[17138]: from='client.15375 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:46.601 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:46 smithi102 ceph-mon[13158]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:46.601 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:46 smithi102 ceph-mon[13158]: from='client.15375 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:48.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:47 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:40:48.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:40:48.261 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:40:48.293 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.293 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.293 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.293 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:48.293 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.294 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.294 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:47 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:48.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-08-31T02:40:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:48 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1792225220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:49.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:48 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1792225220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:48 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1792225220' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:50.114 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:49 smithi070 ceph-mon[12201]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:50.114 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:49 smithi070 ceph-mon[17138]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:50.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:49 smithi102 ceph-mon[13158]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[12201]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:40:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[17138]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:40:52.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:52.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:52.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:52.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:52 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:52.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:52 smithi102 ceph-mon[13158]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-08-31T02:40:52.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:52.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:52.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:52 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:52.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:52 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:53.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:53 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:53.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:53 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:53.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:53 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:53.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:53 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:53.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:53 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:40:53.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:53 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:40:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:40:53] "GET /metrics HTTP/1.1" 200 37914 "" "Prometheus/2.33.4" 2024-08-31T02:40:54.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:54 smithi070 ceph-mon[12201]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:54.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:54 smithi070 ceph-mon[12201]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:54.690 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:54 smithi070 ceph-mon[17138]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:54.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:54 smithi070 ceph-mon[17138]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:54.724 INFO:teuthology.orchestra.run.smithi070.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:54.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:54 smithi102 ceph-mon[13158]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:54.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:54 smithi102 ceph-mon[13158]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-08-31T02:40:55.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:55.965 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[12201]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:55.966 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:55 smithi070 ceph-mon[17138]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:40:56.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:55 smithi102 ceph-mon[13158]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:58.284 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:40:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:58 smithi070 ceph-mon[12201]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:58.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:58 smithi070 ceph-mon[17138]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:58.871 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:58 smithi102 ceph-mon[13158]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-08-31T02:40:58.945 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:40:58.946 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 11s ago 26m - - 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 12s ago 26m - - 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (12m) 11s ago 21m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (12m) 12s ago 21m 54.1M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (3m) 11s ago 20m 51.1M - 3.5 e1d6a67b021e 3b20e62d1fc2 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (15m) 12s ago 25m 504M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (9m) 11s ago 27m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (27m) 11s ago 27m 78.1M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (26m) 12s ago 26m 64.3M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (26m) 11s ago 26m 61.2M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:40:58.948 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (13m) 11s ago 21m 17.5M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (12m) 12s ago 21m 16.5M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (25m) 11s ago 25m 57.3M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (25m) 11s ago 25m 58.7M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (24m) 11s ago 24m 51.5M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (24m) 11s ago 24m 54.1M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (23m) 12s ago 23m 55.0M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (23m) 12s ago 23m 51.4M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (23m) 12s ago 23m 53.9M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (22m) 12s ago 22m 56.4M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (3m) 12s ago 22m 70.1M - 2.33.4 514e6a882f6e 77434e695a34 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (20m) 11s ago 20m 77.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:40:58.949 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (20m) 12s ago 20m 76.5M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:40:59.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:40:59.345 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:40:59.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:40:59] ENGINE Bus STOPPING 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: Failing over to other MGR 2024-08-31T02:40:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/342972590' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: Failing over to other MGR 2024-08-31T02:40:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:40:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/342972590' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='client.25300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:40:59.775 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: Failing over to other MGR 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='mgr.34937 172.21.15.102:0/1478294796' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/342972590' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:40:59.489+0000 7fd133716640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ignoring --setuser ceph since I am not root 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: ignoring --setgroup ceph since I am not root 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-mgr[37911]: -- 172.21.15.102:0/540135375 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f2dcc590e0 con 0x55f2dcc37000 2024-08-31T02:40:59.776 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:40:59.652+0000 7fc62112f040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:40:59.777 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:40:59.711+0000 7fc62112f040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:00.092 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:40:59.865+0000 7fc62112f040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:41:00.093 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:40:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:40:59.936+0000 7fc62112f040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:41:00.093 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:00.091+0000 7fc62112f040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:00.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:00.237+0000 7fc62112f040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:41:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:40:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:41:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:40:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:40:59] ENGINE Bus STOPPED 2024-08-31T02:41:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:00] ENGINE Bus STARTING 2024-08-31T02:41:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:00] ENGINE Serving on http://:::9283 2024-08-31T02:41:00.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:00] ENGINE Bus STARTED 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:41:00.504 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-08-31T02:41:00.505 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:41:00.505 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:41:00.736 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:00.559+0000 7fc62112f040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:41:00.736 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:00.620+0000 7fc62112f040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: mgrmap e37: y(active, starting, since 1.00147s) 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:41:00.737 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: Manager daemon y is now available 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:00.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:41:00.739 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: mgrmap e37: y(active, starting, since 1.00147s) 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:41:00.825 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: Manager daemon y is now available 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:41:00.826 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.34937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: mgrmap e37: y(active, starting, since 1.00147s) 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:41:00.827 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: Manager daemon y is now available 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:00.828 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:41:00.829 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:41:00.990 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:00.808+0000 7fc62112f040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:01.227+0000 7f8c6c1e6640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:41:01] ENGINE Error in HTTPServer.serve 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._connections.run(self.expiration_interval) 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._run(expiration_interval) 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:41:01.501 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s = self.context.wrap_socket( 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.sslsocket_class._create( 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.do_handshake() 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._sslobj.do_handshake() 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:41:01.502 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: mgrmap e38: y(active, since 2s) 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: [31/Aug/2024:02:41:01] ENGINE Bus STARTING 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.503 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: [31/Aug/2024:02:41:01] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: [31/Aug/2024:02:41:01] ENGINE Error in HTTPServer.serve 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:41:01.504 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: [31/Aug/2024:02:41:01] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: [31/Aug/2024:02:41:01] ENGINE Bus STARTED 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[12201]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:01.505 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: mgrmap e38: y(active, since 2s) 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:41:01] ENGINE Bus STARTING 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:41:01] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:41:01] ENGINE Error in HTTPServer.serve 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:41:01.506 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:41:01.507 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:41:01.508 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:41:01.508 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.508 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:41:01] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:41:01.508 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:41:01] ENGINE Bus STARTED 2024-08-31T02:41:01.508 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:01 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:01.747+0000 7fc62112f040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: mgrmap e38: y(active, since 2s) 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:01.751 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: [31/Aug/2024:02:41:01] ENGINE Bus STARTING 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: [31/Aug/2024:02:41:01] ENGINE Serving on https://172.21.15.70:7150 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: [31/Aug/2024:02:41:01] ENGINE Error in HTTPServer.serve 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:41:01.752 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: [31/Aug/2024:02:41:01] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:41:01.753 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: [31/Aug/2024:02:41:01] ENGINE Bus STARTED 2024-08-31T02:41:01.754 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-mon[13158]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:02.035 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:01.840+0000 7fc62112f040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:41:02.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: mgrmap e39: y(active, since 3s) 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: mgrmap e39: y(active, since 3s) 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:02.643+0000 7fc62112f040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: mgrmap e39: y(active, since 3s) 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:41:02.766 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='client.25228 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:02.767 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.062 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:02.770+0000 7fc62112f040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:41:03.062 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:02.923+0000 7fc62112f040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:03.062 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:02.998+0000 7fc62112f040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:41:03.340 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.159+0000 7fc62112f040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:41:03.340 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.233+0000 7fc62112f040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:41:03.340 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.296+0000 7fc62112f040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:41:03.610 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.461+0000 7fc62112f040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:41:03.610 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.533+0000 7fc62112f040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:41:03.610 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.609+0000 7fc62112f040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:03.837 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[12201]: mgrmap e40: y(active, since 5s) 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:03.838 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.839 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:03 smithi070 ceph-mon[17138]: mgrmap e40: y(active, since 5s) 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.705+0000 7fc62112f040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.781+0000 7fc62112f040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:03.845+0000 7fc62112f040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:03 smithi102 ceph-mon[13158]: mgrmap e40: y(active, since 5s) 2024-08-31T02:41:04.940 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:04.941 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:04.669+0000 7fc62112f040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:41:04] ENGINE Bus STARTING 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: CherryPy Checker: 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: The Application mounted at '' has an empty config. 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:41:04] ENGINE Serving on http://:::9283 2024-08-31T02:41:04.942 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[37907]: [31/Aug/2024:02:41:04] ENGINE Bus STARTED 2024-08-31T02:41:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:04.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/74235965' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:04.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:04 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: mgrmap e41: y(active, since 6s), standbys: x 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:41:06.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: mgrmap e41: y(active, since 6s), standbys: x 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.231 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: mgrmap e41: y(active, since 6s), standbys: x 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:06.232 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:07.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:06 smithi070 ceph-mon[12201]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:07.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:06 smithi070 ceph-mon[17138]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:07.369 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:06 smithi102 ceph-mon[13158]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[12201]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:41:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:08.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:08.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[17138]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:41:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:08.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:08 smithi102 ceph-mon[13158]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:41:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:09.187+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:09.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:09.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:09.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:09.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:09.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:09.405 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:09 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.111 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.112 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:10.113 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.114 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.115 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.115 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.115 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.115 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.115 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.116 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.117 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:09 smithi102 ceph-mon[13158]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:41:10.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:10.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:10.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:10.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:10.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:10.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-mon[12201]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:41:10.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.211 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.211 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.211 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.212 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.212 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.212 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.212 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.212 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.213 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:10.214 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:10.215 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:10.216 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.217 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:10.218 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:09 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:41:11.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:11 smithi102 ceph-mon[13158]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:41:11.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:11 smithi102 ceph-mon[13158]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:41:11.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[12201]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:41:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[12201]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:41:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:41:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[17138]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-08-31T02:41:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:12.531 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:12.531 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 bash[58594]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:41:12.814 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.530Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.530Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.530Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:41:12.815 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[54232]: ts=2024-08-31T02:41:12.531Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:41:12.816 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 podman[58605]: 2024-08-31 02:41:12.540908892 +0000 UTC m=+0.105759461 container died 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 podman[58605]: 2024-08-31 02:41:12.905054448 +0000 UTC m=+0.469905059 container cleanup 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 bash[58605]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:41:13.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:12 smithi102 bash[58632]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus.a" found: no such container 2024-08-31T02:41:13.432 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-mon[13158]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:41:13.432 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 podman[58617]: 2024-08-31 02:41:13.172948901 +0000 UTC m=+0.631314382 container remove 77434e695a348ad93d4bd50beef431ac535470ef4e9a55a7548cdbe1e7514e22 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.432 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:41:13.432 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:13.432 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.337s CPU time. 2024-08-31T02:41:13.433 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:13.433 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 podman[58695]: 2024-08-31 02:41:13.30733237 +0000 UTC m=+0.017014071 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-08-31T02:41:13.433 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 podman[58695]: 2024-08-31 02:41:13.432439811 +0000 UTC m=+0.142121503 container create c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.546 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:13 smithi070 ceph-mon[12201]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:41:13.547 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:13 smithi070 ceph-mon[17138]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-08-31T02:41:13.709 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 podman[58695]: 2024-08-31 02:41:13.626294873 +0000 UTC m=+0.335976565 container init c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.709 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 podman[58695]: 2024-08-31 02:41:13.628931242 +0000 UTC m=+0.338612934 container start c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:518 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.651Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.653Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.653Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.654Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.655Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.655Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.436µs 2024-08-31T02:41:13.710 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.655Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:41:13.711 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:13.657Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-08-31T02:41:13.711 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 bash[58695]: c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 2024-08-31T02:41:13.821 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:13] ENGINE Bus STOPPING 2024-08-31T02:41:14.101 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:13 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:14.364 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.130Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-08-31T02:41:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:41:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:14] ENGINE Bus STOPPED 2024-08-31T02:41:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:14] ENGINE Bus STARTING 2024-08-31T02:41:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:14] ENGINE Serving on http://:::9283 2024-08-31T02:41:14.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:41:14] ENGINE Bus STARTED 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.364Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.367Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.370Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.370Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.370Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=43.477µs wal_replay_duration=715.23438ms total_replay_duration=715.2904ms 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.375Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.375Z caller=main.go:947 level=info msg="TSDB started" 2024-08-31T02:41:14.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.375Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:41:14.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.400Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.829032ms db_storage=722ns remote_storage=1.183µs web_handler=360ns query_engine=888ns scrape=628.245µs scrape_sd=110.515µs notify=16.163µs notify_sd=22.045µs rules=23.726632ms 2024-08-31T02:41:14.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:41:14.400Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-08-31T02:41:15.075 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:15.075 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.075 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.075 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.075 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.076 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.076 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.076 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.076 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:14 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:15.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:15.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102.front.sepia.ceph.com:9095"}]: dispatch 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:15.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:14 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: Upgrade: Updating mgr.x 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:16.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:15 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: Upgrade: Updating mgr.x 2024-08-31T02:41:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:16.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: Upgrade: Updating mgr.x 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:16.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:15 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:17.026 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:16 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:17.340 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 podman[59034]: 2024-08-31 02:41:17.026123373 +0000 UTC m=+0.119450090 container died 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True) 2024-08-31T02:41:17.591 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:17 smithi070 ceph-mon[12201]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:17.591 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:17 smithi070 ceph-mon[17138]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:17 smithi102 ceph-mon[13158]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:17.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 podman[59034]: 2024-08-31 02:41:17.369788226 +0000 UTC m=+0.463114940 container cleanup 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-31T02:41:17.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 bash[59034]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:41:17.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 podman[59046]: 2024-08-31 02:41:17.579443608 +0000 UTC m=+0.550759081 container remove 09b8094f1394249ad6c679f6d2207637654eb4f7f3d0fc2f12a146dc4f27982d (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-31T02:41:17.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:41:18.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-31T02:41:18.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 systemd[1]: Stopped Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:18.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 37.661s CPU time. 2024-08-31T02:41:18.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:17 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:18.450 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 podman[59180]: 2024-08-31 02:41:18.10207756 +0000 UTC m=+0.016209978 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:41:18.450 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 podman[59180]: 2024-08-31 02:41:18.213863086 +0000 UTC m=+0.127995546 container create ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True) 2024-08-31T02:41:18.450 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 podman[59180]: 2024-08-31 02:41:18.450045924 +0000 UTC m=+0.364178352 container init ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:41:18.786 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 podman[59180]: 2024-08-31 02:41:18.453082193 +0000 UTC m=+0.367214625 container start ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-31T02:41:18.786 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-mgr[59199]: -- 172.21.15.102:0/1782130262 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56183781f0e0 con 0x5618377fd000 2024-08-31T02:41:18.786 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 bash[59180]: ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be 2024-08-31T02:41:18.786 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:18.786 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:18.574+0000 7f81bee53040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:41:18.787 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:18.633+0000 7f81bee53040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:18.787 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:18.785+0000 7f81bee53040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:41:19.135 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:18.851+0000 7f81bee53040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:41:19.135 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:18.997+0000 7f81bee53040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:19.135 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:19.134+0000 7f81bee53040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:41:19.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:19.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.367 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:19.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:19.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:19.368 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:19.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:19.449+0000 7f81bee53040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:41:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:19.510+0000 7f81bee53040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:41:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:19.695+0000 7f81bee53040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:20.598 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:20 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:20.646 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:20 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:20.646 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:20 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:20.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:20 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:20.647 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:20.578+0000 7f81bee53040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:41:20.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:20.645+0000 7f81bee53040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:41:21.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-mon[12201]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:21.606 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-mon[17138]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:21.615 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-mon[13158]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:21.615 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.353+0000 7f81bee53040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:41:21.616 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.472+0000 7f81bee53040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:41:21.616 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.614+0000 7f81bee53040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:21.895 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.675+0000 7f81bee53040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:41:21.895 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.824+0000 7f81bee53040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:41:21.896 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.893+0000 7f81bee53040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:21.763+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:21.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:21.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:21.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:22.167 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:21.954+0000 7f81bee53040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:41:22.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.109+0000 7f81bee53040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:41:22.443 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.175+0000 7f81bee53040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.240+0000 7f81bee53040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.323+0000 7f81bee53040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.383+0000 7f81bee53040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:22.442+0000 7f81bee53040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/786435252' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:22.444 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:22.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:22.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/786435252' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:41:22.636 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:22.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/786435252' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:22.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:22.639 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:23.277+0000 7f81bee53040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: [31/Aug/2024:02:41:23] ENGINE Bus STARTING 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: CherryPy Checker: 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: The Application mounted at '' has an empty config. 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.375 INFO:journalctl@ceph.mon.b.smithi102.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.376 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.377 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:23.378 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.379 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.380 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all mgr 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:23.381 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:23.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.435 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.436 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.437 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:23.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:23.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi-foo-smithi070-ashysy 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: /usr/bin/podman: stderr Error: no such container ceph-862b0c56-673e-11ef-bcd4-c7b262605968-iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.441 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all mgr 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:23.442 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.443 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all mgr 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:23.444 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/1026387984' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:23.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: [31/Aug/2024:02:41:23] ENGINE Serving on http://:::9283 2024-08-31T02:41:23.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[59195]: [31/Aug/2024:02:41:23] ENGINE Bus STARTED 2024-08-31T02:41:23.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:23.487+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:23.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:23.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:23.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: mgrmap e42: y(active, since 24s), standbys: x 2024-08-31T02:41:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: mgrmap e42: y(active, since 24s), standbys: x 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: mgrmap e42: y(active, since 24s), standbys: x 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:24.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:25.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:25.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:25.578 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[12201]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:25.581 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:25.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[17138]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:25.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:25.585 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:25.585 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:25.424+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.585 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:25.585 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.586 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:25.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.589 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.590 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.590 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.590 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.590 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:41:25] "GET /metrics HTTP/1.1" 200 37912 "" "Prometheus/2.33.4" 2024-08-31T02:41:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:25 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:25.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:25.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:25 smithi102 ceph-mon[13158]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:25 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:25 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:25.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:25 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:26.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:26.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:26.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:26.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:26.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:27.031+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:27.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:27.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.207 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:27 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.601 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.601 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.601 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.601 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.602 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:27.603 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.604 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.605 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.607 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.608 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:27.609 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:27.610 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:27.611 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:27.612 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:27.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:27.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:27.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:27.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:28.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.485 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.487 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:28.488 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:28.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:28.491 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:28.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.492 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-mon[12201]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:29.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-mon[17138]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:29.464 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:29 smithi102 ceph-mon[13158]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:30.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:29.944+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:30.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:30.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:30.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:30.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:30.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:30.881 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:41:31.566 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 0s ago 27m - - 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 3s ago 26m - - 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (13m) 0s ago 21m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (13m) 3s ago 21m 54.5M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 running (3m) 0s ago 20m 51.5M - 3.5 e1d6a67b021e 3b20e62d1fc2 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283 running (16m) 3s ago 26m 354M - 19.1.1-240-g34b9d27b 72d622302cc2 09b8094f1394 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (10m) 0s ago 27m 498M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (28m) 0s ago 28m 83.3M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (26m) 3s ago 26m 67.1M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (26m) 0s ago 26m 63.3M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (13m) 0s ago 22m 20.3M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:41:31.567 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (13m) 3s ago 22m 16.4M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (26m) 0s ago 26m 57.4M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (25m) 0s ago 25m 58.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (25m) 0s ago 25m 50.5M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (24m) 0s ago 24m 53.2M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (24m) 3s ago 24m 56.7M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (24m) 3s ago 24m 51.2M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (23m) 3s ago 23m 54.0M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (23m) 3s ago 23m 57.8M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (3m) 3s ago 22m 72.9M - 2.33.4 514e6a882f6e 77434e695a34 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (20m) 0s ago 20m 78.0M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:41:31.568 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (20m) 3s ago 20m 76.7M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:41:31.657 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:31 smithi102 ceph-mon[13158]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:31 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:31.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:31.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.662 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[12201]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[17138]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:31.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:31.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:31.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:31.976 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:41:31.976 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:41:31.976 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:41:31.977 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:41:31.978 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:41:31.978 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mgr daemons", 2024-08-31T02:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:41:32.578 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[12201]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[12201]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.579 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1984337370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:32.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[17138]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[17138]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:32 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1984337370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:32 smithi102 ceph-mon[13158]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:32 smithi102 ceph-mon[13158]: from='client.15486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:32 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1984337370' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:33.583 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:33.583 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[12201]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:33.583 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[12201]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:33.583 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[12201]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:33.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:33.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[17138]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:33.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[17138]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:33.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:33 smithi070 ceph-mon[17138]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:33 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:33 smithi102 ceph-mon[13158]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:33 smithi102 ceph-mon[13158]: from='client.15492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:33 smithi102 ceph-mon[13158]: from='client.15504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:41:34.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:34.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:34.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: Upgrade: Updating mgr.x 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: Upgrade: Updating mgr.x 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: Upgrade: Updating mgr.x 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:35 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:41:35] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-08-31T02:41:36.279 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:36.605 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 podman[60759]: 2024-08-31 02:41:36.293618164 +0000 UTC m=+0.138273790 container died ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 677 B/s rd, 0 op/s 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 677 B/s rd, 0 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 908 B/s rd, 0 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 677 B/s rd, 0 op/s 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:41:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:36.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 podman[60759]: 2024-08-31 02:41:36.704586621 +0000 UTC m=+0.549242244 container cleanup ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:41:36.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 bash[60759]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:41:37.202 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 podman[60797]: 2024-08-31 02:41:36.939393618 +0000 UTC m=+0.642318374 container remove ea1e7d39e739c55876fbc14e26e937132662e924c2ca2d87e2f09043fd0e70be (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-31T02:41:37.203 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:36 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:41:37.203 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-31T02:41:37.203 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 systemd[1]: Stopped Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:37.203 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 5.757s CPU time. 2024-08-31T02:41:37.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:37 smithi070 ceph-mon[12201]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:37.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:37 smithi070 ceph-mon[12201]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:37.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:37 smithi070 ceph-mon[17138]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:37.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:37 smithi070 ceph-mon[17138]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:37.588 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:37 smithi102 ceph-mon[13158]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:37.588 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:37 smithi102 ceph-mon[13158]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:37.588 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:37.868 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 podman[60926]: 2024-08-31 02:41:37.491201264 +0000 UTC m=+0.016342141 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:41:37.869 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 podman[60926]: 2024-08-31 02:41:37.631938426 +0000 UTC m=+0.157079332 container create a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, io.openshift.expose-services=, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, version=8, ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-08-31T02:41:38.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 podman[60926]: 2024-08-31 02:41:37.868522305 +0000 UTC m=+0.393663170 container init a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, release=754, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:41:38.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 podman[60926]: 2024-08-31 02:41:37.871953832 +0000 UTC m=+0.397094705 container start a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git) 2024-08-31T02:41:38.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 ceph-mgr[60944]: -- 172.21.15.102:0/2722981823 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555fd052a340 con 0x555fd0502800 2024-08-31T02:41:38.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 bash[60926]: a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc 2024-08-31T02:41:38.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:37 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:38.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.011+0000 7f751de5c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:41:38.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.095+0000 7f751de5c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:38.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.273+0000 7f751de5c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:41:38.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.360+0000 7f751de5c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:41:38.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.488+0000 7f751de5c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:39.022 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:38 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:38.694+0000 7f751de5c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:41:39.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:39.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:39.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:39.021+0000 7f751de5c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:39.110+0000 7f751de5c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.282 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:39.283 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:39.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:39.281+0000 7f751de5c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:41:39.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:39.653+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:39.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:39.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:39.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:39.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:39.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:39.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:39.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:39.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:39.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:39.957 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:39.958 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:40.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:40.320 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.353 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.353 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:40.353 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:40.353 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.353 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:40.354 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:40.354 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:40.354 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:40.663 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:40.352+0000 7f751de5c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:41:40.663 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:40.446+0000 7f751de5c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:41:41.625 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:41.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.628 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.629 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:41.630 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:41.376+0000 7f751de5c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:41.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:41.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:41.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:41.387+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:41.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:41.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:41.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:41.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:41.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:41.729+0000 7f751de5c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:41:41.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:41.866+0000 7f751de5c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:42.352 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:41.960+0000 7f751de5c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:41:42.353 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.071+0000 7f751de5c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:41:42.353 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.182+0000 7f751de5c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.611 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:42.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.363+0000 7f751de5c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.467+0000 7f751de5c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.571+0000 7f751de5c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:42.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:42.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:42.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:42.617 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:42.617 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:42.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:42.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:42.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:42.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:42.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.870+0000 7f751de5c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:43.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:42.961+0000 7f751de5c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:41:43.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:43.054+0000 7f751de5c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:41:43.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:42.910+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:43.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:43.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:43.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:43.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:43.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:43.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 992 B/s rd, 0 op/s 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:43.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:44.291 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:44.183+0000 7f751de5c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: [31/Aug/2024:02:41:44] ENGINE Bus STARTING 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: CherryPy Checker: 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: The Application mounted at '' has an empty config. 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: [31/Aug/2024:02:41:44] ENGINE Serving on http://:::9283 2024-08-31T02:41:44.292 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[60940]: [31/Aug/2024:02:41:44] ENGINE Bus STARTED 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.450 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.451 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:44.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:44.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:44.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:44.459 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.460 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.461 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:44.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:44.463 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:44.463 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:44.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:44.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:41:44.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:44 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:44.665+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:44.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:44.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:44.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:44.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:44.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:44.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:44.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:45.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: mgrmap e43: y(active, since 45s), standbys: x 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:45.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:45.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: mgrmap e43: y(active, since 45s), standbys: x 2024-08-31T02:41:45.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:45.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:45.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: mgrmap e43: y(active, since 45s), standbys: x 2024-08-31T02:41:45.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:45.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:45.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:41:45.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:41:45] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:46.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:46.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:46.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:46.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:46 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:46.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:46.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:46.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:46.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:46.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:46.409+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:46.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:46.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:46.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:46 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:47.595 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:47.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:47.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:48.017+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:48.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.646 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.647 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:48.648 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.649 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.650 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:48.651 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:48.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.656 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:48.658 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:48.661 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:48.662 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:48.663 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:48.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:48.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:49.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:49.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:49.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:49.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:49.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:49 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:49.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:49.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1002 B/s rd, 0 op/s 2024-08-31T02:41:49.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:49.600+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:49.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:49.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:49.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:49.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:50.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:50.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:50 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:50.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:51.352 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:51.196+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.352 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:51.352 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.353 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:51.354 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:51.355 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.356 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.357 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.446 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.447 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:51.448 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.449 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.450 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.451 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:51 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:51.654 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:51.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:51.659 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.660 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.661 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:51.662 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:51.663 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:51.664 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:51.665 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.666 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:41:51.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:51 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:52 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:52.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.552 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:52 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.653 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:52.654 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:52 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:52.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:41:52.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:52.661 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:53 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:53 smithi102 ceph-mon[13158]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-08-31T02:41:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:53 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:53 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:53 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:53.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[12201]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-08-31T02:41:53.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:41:53.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[17138]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1 op/s 2024-08-31T02:41:53.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:53.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:53 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:54.645 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:54.645 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:54.646 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:41:54.647 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:55 smithi102 ceph-mon[13158]: Upgrade: Updating mgr.x 2024-08-31T02:41:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:55 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:55 smithi070 ceph-mon[12201]: Upgrade: Updating mgr.x 2024-08-31T02:41:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:55 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:55.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:41:55] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.33.4" 2024-08-31T02:41:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:55 smithi070 ceph-mon[17138]: Upgrade: Updating mgr.x 2024-08-31T02:41:55.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:55 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:41:56.356 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:56 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:56.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:56 smithi102 ceph-mon[13158]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-08-31T02:41:56.659 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:56 smithi102 podman[62763]: 2024-08-31 02:41:56.355756507 +0000 UTC m=+0.120730176 container died a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, version=8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, RELEASE=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-08-31T02:41:56.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:56 smithi070 ceph-mon[12201]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-08-31T02:41:56.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:56 smithi070 ceph-mon[17138]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 976 B/s rd, 0 op/s 2024-08-31T02:41:56.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:56 smithi102 podman[62763]: 2024-08-31 02:41:56.741578468 +0000 UTC m=+0.506552127 container cleanup a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.buildah.version=1.19.8, release=754, RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_CLEAN=True) 2024-08-31T02:41:56.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:56 smithi102 bash[62763]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:41:57.359 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 podman[62775]: 2024-08-31 02:41:57.018539394 +0000 UTC m=+0.663580691 container remove a76647cf5f6574eeb1bf64972a93377a536de4cb815bd195fdec42877b87f8bc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, RELEASE=HEAD, ceph=True, io.openshift.expose-services=) 2024-08-31T02:41:57.359 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:41:57.359 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-31T02:41:57.359 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: Stopped Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:57.359 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 7.704s CPU time. 2024-08-31T02:41:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:57 smithi102 ceph-mon[13158]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:57.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:41:57.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 podman[62901]: 2024-08-31 02:41:57.472063827 +0000 UTC m=+0.016090661 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:41:57.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 podman[62901]: 2024-08-31 02:41:57.594337466 +0000 UTC m=+0.138364288 container create 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-31T02:41:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:57 smithi070 ceph-mon[12201]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:57 smithi070 ceph-mon[17138]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 podman[62901]: 2024-08-31 02:41:57.830838048 +0000 UTC m=+0.374864875 container init 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 podman[62901]: 2024-08-31 02:41:57.833594167 +0000 UTC m=+0.377620992 container start 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 ceph-mgr[62919]: -- 172.21.15.102:0/1018564691 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b1298ad0e0 con 0x55b12988a800 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 bash[62901]: 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:57.959+0000 7f1aedb48040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:41:58.031 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.030+0000 7f1aedb48040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:41:58.397 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.181+0000 7f1aedb48040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:41:58.397 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.247+0000 7f1aedb48040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:41:58.397 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.396+0000 7f1aedb48040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:41:58.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.536+0000 7f1aedb48040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:41:59.105 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.854+0000 7f1aedb48040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:58.915+0000 7f1aedb48040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:41:59.100+0000 7f1aedb48040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:41:59.106 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.107 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:41:59 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:59.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 887 B/s rd, 0 op/s 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:41:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:41:59.606+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:41:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:41:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:41:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:41:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:41:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:41:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:00.303 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:00.037+0000 7f1aedb48040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:00.107+0000 7f1aedb48040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:00.304 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:00.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:00.305 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:00.342 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.342 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:00.342 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:00.342 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:00.342 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:00.343 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:00.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:00.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:00.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:00.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:00.344 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.087 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:01.088 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:01.089 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:01.090 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.091 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:00.829+0000 7f1aedb48040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:00.944+0000 7f1aedb48040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.085+0000 7f1aedb48040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:01.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:01.345 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.147+0000 7f1aedb48040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:42:01.346 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.315+0000 7f1aedb48040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:42:01.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:01.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.431 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.432 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:01.433 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.434 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.436 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:01.437 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.438 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.439 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:01.440 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:01.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 791 B/s rd, 0 op/s 2024-08-31T02:42:01.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:01.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:01.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:01.441 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:01.085+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.442 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:01.443 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:01.444 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:01.445 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:01.446 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:01.608 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.382+0000 7f1aedb48040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:42:01.609 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.445+0000 7f1aedb48040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:42:01.609 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.607+0000 7f1aedb48040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:42:01.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.675+0000 7f1aedb48040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:42:01.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.742+0000 7f1aedb48040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:42:01.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.825+0000 7f1aedb48040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:42:01.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.885+0000 7f1aedb48040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:42:02.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:01.944+0000 7f1aedb48040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:42:02.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:02.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:02.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:02.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.408 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.409 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:02.410 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:02.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:02.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:02.525+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:02.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:02.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:02.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:02.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:02.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:02.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:02.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:02.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:02.706 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:02.801+0000 7f1aedb48040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: [31/Aug/2024:02:42:02] ENGINE Bus STARTING 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: CherryPy Checker: 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: The Application mounted at '' has an empty config. 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: [31/Aug/2024:02:42:02] ENGINE Serving on http://:::9283 2024-08-31T02:42:02.909 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[62915]: [31/Aug/2024:02:42:02] ENGINE Bus STARTED 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:03.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:03.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:03.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:03 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.299 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.300 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:03.301 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:03 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/2117220304' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 10s ago 27m - - 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 13s ago 27m - - 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (14m) 10s ago 22m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (13m) 13s ago 22m 53.3M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 10s ago 21m 44.3M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 starting - - - - 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (11m) 10s ago 28m 501M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (28m) 10s ago 28m 85.7M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (27m) 13s ago 27m 69.6M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:42:03.414 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (27m) 10s ago 27m 66.3M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (14m) 10s ago 22m 19.0M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (13m) 13s ago 22m 21.5M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (26m) 10s ago 26m 57.4M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (26m) 10s ago 26m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (25m) 10s ago 25m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (25m) 10s ago 25m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (24m) 13s ago 24m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (24m) 13s ago 24m 51.5M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (24m) 13s ago 24m 54.3M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (23m) 13s ago 23m 57.9M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (50s) 13s ago 23m 62.1M - 2.33.4 514e6a882f6e c953e8b1c53a 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (21m) 10s ago 21m 78.2M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:42:03.415 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (21m) 13s ago 21m 77.0M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:42:03.821 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:42:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:42:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:42:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:42:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:42:03.823 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.082 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:04.083 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:04.084 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.085 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.086 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: mgrmap e44: y(active, since 65s), standbys: x 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3944739701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:04.087 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.088 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:04.089 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.090 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.091 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: mgrmap e44: y(active, since 65s), standbys: x 2024-08-31T02:42:04.092 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3944739701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:04.165 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:42:04.165 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:42:04.165 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:42:04.165 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:42:04.166 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:42:04.166 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "", 2024-08-31T02:42:04.166 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mgr daemons", 2024-08-31T02:42:04.166 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:42:04.166 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:42:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:04.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: Upgrade: 1 mgr daemon(s) are 17.2.0 != target 19.1.1-240-g34b9d27b 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: from='client.25432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: mgrmap e44: y(active, since 65s), standbys: x 2024-08-31T02:42:04.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:04 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3944739701' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:04.410+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.682 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.683 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:04.684 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:04.685 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:04.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:04.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:04.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:04.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:04.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='client.25438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:05.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:05.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:05.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='client.25438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='client.25438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:05.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:05.876 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:05] "GET /metrics HTTP/1.1" 200 37983 "" "Prometheus/2.33.4" 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:05.980+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.489 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.490 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:06.491 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.492 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.493 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:06.494 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.495 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.496 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.497 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.498 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:06.499 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:06.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:06.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:07.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.610 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.610 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.610 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.611 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:07.612 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.613 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.614 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:07.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:07.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:07.621 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:07.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:07.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:07 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:07.665+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:08.608 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.608 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:08.609 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:08.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:08.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:08.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:09.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:09.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:09 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:09.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:09.374+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:09.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:10.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:10.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.631 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.632 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:10.633 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:10.634 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:10.635 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:10.636 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:10.954+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:11.153 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:11.154 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:11.155 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:11.156 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:11.157 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:11.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:11.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:11.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:11.158 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:11.402 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.404 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:11.502 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:11.503 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:11.504 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:12.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:12.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:12.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:12.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:12.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:13.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:42:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:42:13.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:13.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:13.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:14 smithi102 ceph-mon[13158]: Upgrade: Updating mgr.x 2024-08-31T02:42:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:14 smithi102 ceph-mon[13158]: Deploying daemon mgr.x on smithi102 2024-08-31T02:42:14.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:14 smithi102 ceph-mon[13158]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:14.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:14 smithi102 ceph-mon[13158]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[12201]: Upgrade: Updating mgr.x 2024-08-31T02:42:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[12201]: Deploying daemon mgr.x on smithi102 2024-08-31T02:42:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[12201]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[12201]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[17138]: Upgrade: Updating mgr.x 2024-08-31T02:42:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[17138]: Deploying daemon mgr.x on smithi102 2024-08-31T02:42:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[17138]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:14 smithi070 ceph-mon[17138]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:42:15.472 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:15.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:15.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:15.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:15] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-08-31T02:42:16.033 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:15 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:16.311 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 podman[64638]: 2024-08-31 02:42:16.032387547 +0000 UTC m=+0.131717997 container died 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-08-31T02:42:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:16 smithi102 ceph-mon[13158]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:16.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 podman[64638]: 2024-08-31 02:42:16.401166669 +0000 UTC m=+0.500497113 container cleanup 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27) 2024-08-31T02:42:16.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 bash[64638]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:42:16.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:16 smithi070 ceph-mon[12201]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:16 smithi070 ceph-mon[17138]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:16.937 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 podman[64651]: 2024-08-31 02:42:16.685554686 +0000 UTC m=+0.649957820 container remove 3c474cfc076baa725ac18f8c814e0d0263fc45923eb39d70c155ed89ed7381ef (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:42:16.937 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:42:16.937 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-31T02:42:16.937 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 systemd[1]: Stopped Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:16.938 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:16 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service: Consumed 5.885s CPU time. 2024-08-31T02:42:17.248 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 systemd[1]: Starting Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:17.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-mon[13158]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:42:17.517 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 podman[64781]: 2024-08-31 02:42:17.150655291 +0000 UTC m=+0.015964371 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:42:17.517 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 podman[64781]: 2024-08-31 02:42:17.253872058 +0000 UTC m=+0.119181137 container create d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True) 2024-08-31T02:42:17.518 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 podman[64781]: 2024-08-31 02:42:17.489960585 +0000 UTC m=+0.355269665 container init d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-31T02:42:17.518 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 podman[64781]: 2024-08-31 02:42:17.493007637 +0000 UTC m=+0.358316719 container start d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-31T02:42:17.518 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-mgr[64799]: -- 172.21.15.102:0/3590904044 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bdfae810e0 con 0x55bdfae5f400 2024-08-31T02:42:17.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:17 smithi070 ceph-mon[12201]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:42:17.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:17 smithi070 ceph-mon[17138]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:42:17.787 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 bash[64781]: d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc 2024-08-31T02:42:17.787 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 systemd[1]: Started Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:17.788 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:17.615+0000 7fcec87fc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:42:17.788 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:17.683+0000 7fcec87fc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:42:18.060 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:17.839+0000 7fcec87fc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:42:18.060 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:17.906+0000 7fcec87fc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:42:18.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:18.066+0000 7fcec87fc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:42:18.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:18.213+0000 7fcec87fc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:42:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:18 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:18.804 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:18.542+0000 7fcec87fc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:42:18.804 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:18.606+0000 7fcec87fc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 929 B/s rd, 0 op/s 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:18.805 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:18.806 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:18.806 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:18.806 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:19.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:18 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:18.802+0000 7fcec87fc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:19.239+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:19.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:19.708+0000 7fcec87fc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:42:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:19.776+0000 7fcec87fc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:42:19.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:19.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:19.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:19.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:19.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:19.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:19.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:19.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:19.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:19.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.930 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.931 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:19 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.950 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.951 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:19.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:19.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.955 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.956 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:19.957 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:19.958 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:19.959 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:19.960 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 855 B/s rd, 0 op/s 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:19.961 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:19 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:20.784 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:20.524+0000 7fcec87fc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:42:20.784 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:20.638+0000 7fcec87fc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:42:20.784 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:20.783+0000 7fcec87fc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:42:21.071 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:20.845+0000 7fcec87fc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:42:21.071 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:20 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:20.998+0000 7fcec87fc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:42:21.071 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.070+0000 7fcec87fc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:20.906+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:21.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:21.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:21.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:21.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:21.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:21.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:21.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:21.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:21.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:21.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:21.355 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.130+0000 7fcec87fc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:42:21.355 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.299+0000 7fcec87fc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:21.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:21.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.366+0000 7fcec87fc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:42:21.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.436+0000 7fcec87fc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:42:21.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.521+0000 7fcec87fc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:42:21.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.580+0000 7fcec87fc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:42:21.645 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:21 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:21.643+0000 7fcec87fc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:21.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:21.650 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:22.558 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.559 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:22.560 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.561 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.562 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:22.563 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:22.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:22.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:22.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:42:22.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:22.558+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:22.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:22.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:22.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:22.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:22.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:22.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:22.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:22.897 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:22.557+0000 7fcec87fc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:42:22.897 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:42:22] ENGINE Bus STARTING 2024-08-31T02:42:22.897 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: CherryPy Checker: 2024-08-31T02:42:22.897 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: The Application mounted at '' has an empty config. 2024-08-31T02:42:22.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:42:22.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:42:22] ENGINE Serving on http://:::9283 2024-08-31T02:42:22.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:42:22 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:42:22] ENGINE Bus STARTED 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: Standby manager daemon x restarted 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: Standby manager daemon x started 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:23.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:23.619 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: Standby manager daemon x restarted 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: Standby manager daemon x started 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:42:23.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/103900893' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:24.172+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.370 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:24.371 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:24.372 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.373 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.374 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.375 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.456 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.457 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:24.458 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.459 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.460 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: mgrmap e45: y(active, since 84s), standbys: x 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:24.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:24.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:24.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:24.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:24 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:24.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:24.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:24.694 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:24.695 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:24.696 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:24.697 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: mgrmap e45: y(active, since 84s), standbys: x 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:24.698 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: mgrmap e45: y(active, since 84s), standbys: x 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:24.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:24 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.547 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:25.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:25.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:25.551 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:25.552 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:25 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:25.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:25] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:25.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:25.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:25.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:25.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:25.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:25.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:26.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:25.905+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:26.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:26.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:26.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:26.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:26.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:26.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:26.661 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:26.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:26.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:27.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:27.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:27 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:27.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:27.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:27.571+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:27.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:27.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:27.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:28.640 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.640 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:28.640 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.640 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:28.640 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:28.641 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:28.641 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:28.641 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:28.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:28.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:28.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:28.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:29.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:29.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:29.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:29 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:29.374+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:29.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:29.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:29.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:30.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.452 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:30.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:30.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:30.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:30.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:30.477 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:30.478 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:30.479 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:30.480 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:30.481 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:30.482 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:30.482 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.482 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:30.482 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.497 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.498 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.499 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.500 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:31.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:31.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:31.243+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:31.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:31.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:31.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:31.707 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:31.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all crash 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all mds 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all nfs 2024-08-31T02:42:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:32 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:42:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:32.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:32.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:32.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all crash 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all mds 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all nfs 2024-08-31T02:42:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all crash 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all mds 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all nfs 2024-08-31T02:42:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:32 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:42:33.543 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:33.543 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:33 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:34.556 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:42:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:34 smithi102 ceph-mon[13158]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[12201]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[17138]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:34.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:35.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:34 smithi070 systemd[1]: Stopping Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:35.256 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 1s ago 28m - - 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 4s ago 28m - - 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (14m) 1s ago 22m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (14m) 4s ago 22m 53.6M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 1s ago 21m 44.7M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (18s) 4s ago 27m 441M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (11m) 1s ago 29m 505M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (29m) 1s ago 29m 88.7M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (27m) 4s ago 27m 69.8M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (27m) 1s ago 27m 67.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (14m) 1s ago 23m 18.0M - 1.3.1 1dbe0e931976 a4fe61211483 2024-08-31T02:42:35.257 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (14m) 4s ago 23m 18.7M - 1.3.1 1dbe0e931976 a53d3e4c453e 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (27m) 1s ago 27m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (26m) 1s ago 26m 58.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (26m) 1s ago 26m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (25m) 1s ago 25m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (25m) 4s ago 25m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (25m) 4s ago 25m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (24m) 4s ago 24m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (24m) 4s ago 24m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (81s) 4s ago 23m 64.7M - 2.33.4 514e6a882f6e c953e8b1c53a 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (21m) 1s ago 21m 78.5M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:42:35.258 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (21m) 4s ago 21m 77.6M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:42:35.573 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:35 smithi070 podman[82061]: 2024-08-31 02:42:35.268827064 +0000 UTC m=+0.116994349 container died a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:35.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[12201]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[12201]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[12201]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[17138]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[17138]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[17138]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:35.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:35.667 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:42:35.668 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:42:35.669 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:42:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:35 smithi102 ceph-mon[13158]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:35 smithi102 ceph-mon[13158]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:35 smithi102 ceph-mon[13158]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:35 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:35] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-08-31T02:42:35.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:35 smithi070 podman[82061]: 2024-08-31 02:42:35.783471958 +0000 UTC m=+0.631639243 container cleanup a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:35.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:35 smithi070 bash[82061]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:42:36.008 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading node-exporter daemons", 2024-08-31T02:42:36.009 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:42:36.009 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:42:36.270 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 podman[82075]: 2024-08-31 02:42:36.035175163 +0000 UTC m=+0.765405730 container remove a4fe61211483e15cc6fbe08a7d168edda2b4c38b041713c73c66a4de9af5f26e (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:36.270 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:42:36.270 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-31T02:42:36.270 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 systemd[1]: Stopped Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:36.270 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 5.296s CPU time. 2024-08-31T02:42:36.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 systemd[1]: Starting Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:36.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[12201]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[12201]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[12201]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.648 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/97181492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:36.648 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[17138]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[17138]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[17138]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:36 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/97181492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:36.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:36 smithi102 ceph-mon[13158]: from='client.15552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:36 smithi102 ceph-mon[13158]: from='client.15558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:36 smithi102 ceph-mon[13158]: from='client.25474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:36 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/97181492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:36.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:36 smithi070 bash[82248]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-31T02:42:37.397 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:37 smithi070 bash[82248]: Getting image source signatures 2024-08-31T02:42:37.398 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:37 smithi070 bash[82248]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-31T02:42:37.398 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:37 smithi070 bash[82248]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-31T02:42:37.398 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:37 smithi070 bash[82248]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-31T02:42:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:37 smithi102 ceph-mon[13158]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-08-31T02:42:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:37 smithi102 ceph-mon[13158]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:37 smithi070 ceph-mon[17138]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-08-31T02:42:37.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:37 smithi070 ceph-mon[17138]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:37.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:37 smithi070 ceph-mon[12201]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 978 B/s rd, 0 op/s 2024-08-31T02:42:37.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:37 smithi070 ceph-mon[12201]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:42:38.699 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:38 smithi070 bash[82248]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-31T02:42:38.699 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:38 smithi070 bash[82248]: Writing manifest to image destination 2024-08-31T02:42:39.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:38 smithi070 podman[82248]: 2024-08-31 02:42:38.816486839 +0000 UTC m=+2.261195944 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-31T02:42:39.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 podman[82248]: 2024-08-31 02:42:39.065240543 +0000 UTC m=+2.509949680 container create ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:39.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-mon[12201]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:42:39.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-mon[17138]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 podman[82248]: 2024-08-31 02:42:39.35080146 +0000 UTC m=+2.795510554 container init ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 podman[82248]: 2024-08-31 02:42:39.353829878 +0000 UTC m=+2.798538978 container start ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.358Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.358Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.358Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.358Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:42:39.460 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=arp 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-31T02:42:39.461 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=edac 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-31T02:42:39.462 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=os 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-31T02:42:39.463 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=stat 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=time 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=uname 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.359Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.360Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[82323]: ts=2024-08-31T02:42:39.360Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-31T02:42:39.464 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 bash[82248]: ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 2024-08-31T02:42:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:39 smithi102 ceph-mon[13158]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:42:39.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:39 smithi070 systemd[1]: Started Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:40.861 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:40 smithi102 ceph-mon[13158]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:42:40.861 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:40.861 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:40 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:40.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[12201]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:42:40.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:40.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:40.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[17138]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2024-08-31T02:42:40.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:40.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:40 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:41.518 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:41 smithi102 systemd[1]: Stopping Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:41.918 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:41 smithi102 podman[66687]: 2024-08-31 02:42:41.676092578 +0000 UTC m=+0.119545483 container died a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:41 smithi102 ceph-mon[13158]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:42:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:41 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:42:41.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:41 smithi070 ceph-mon[12201]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:42:41.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:41 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:42:41.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:41 smithi070 ceph-mon[17138]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:42:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:41 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:42:42.417 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 podman[66687]: 2024-08-31 02:42:42.132310119 +0000 UTC m=+0.575763027 container cleanup a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:42.417 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 bash[66687]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-31T02:42:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:42 smithi102 ceph-mon[13158]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:42 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:42.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 podman[66699]: 2024-08-31 02:42:42.417001915 +0000 UTC m=+0.740632711 container remove a53d3e4c453ec2c554baa95e6a229d6ff0d09c8e88b770ca3dd157c4662fce7a (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:42.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:42:42.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-31T02:42:42.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 systemd[1]: Stopped Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:42.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 5.467s CPU time. 2024-08-31T02:42:42.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:42 smithi070 ceph-mon[12201]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:42.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:42 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:42.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:42 smithi070 ceph-mon[17138]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:42.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:42 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:43.168 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 systemd[1]: Starting Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:43.168 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:42 smithi102 bash[66826]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-31T02:42:43.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:43 smithi102 bash[66826]: Getting image source signatures 2024-08-31T02:42:43.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:43 smithi102 bash[66826]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-31T02:42:43.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:43 smithi102 bash[66826]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-31T02:42:43.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:43 smithi102 bash[66826]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-31T02:42:44.848 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:44 smithi102 ceph-mon[13158]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:42:44.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:44 smithi070 ceph-mon[12201]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:42:45.005 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:44 smithi070 ceph-mon[17138]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:42:45.149 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:44 smithi102 bash[66826]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-31T02:42:45.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 bash[66826]: Writing manifest to image destination 2024-08-31T02:42:45.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:45.721 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 podman[66826]: 2024-08-31 02:42:45.40454905 +0000 UTC m=+2.561677573 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-31T02:42:45.722 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 podman[66826]: 2024-08-31 02:42:45.495057518 +0000 UTC m=+2.652186086 container create 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:45.722 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 podman[66826]: 2024-08-31 02:42:45.713676737 +0000 UTC m=+2.870805250 container init 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:45.722 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 podman[66826]: 2024-08-31 02:42:45.717532531 +0000 UTC m=+2.874661044 container start 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:42:45.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:45.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:45] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-08-31T02:42:45.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:42:46.168 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.721Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-31T02:42:46.168 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.721Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=arp 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-31T02:42:46.169 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=edac 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-31T02:42:46.170 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.722Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-31T02:42:46.171 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=os 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-31T02:42:46.172 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=stat 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=time 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=uname 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-31T02:42:46.173 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[66901]: ts=2024-08-31T02:42:45.723Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-31T02:42:46.174 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 bash[66826]: 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 2024-08-31T02:42:46.174 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:42:45 smithi102 systemd[1]: Started Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:42:46.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:46.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:46.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:46.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:46.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:46 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:47.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:47.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:47.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:46 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:47.476+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:47.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:47.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:47.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:47.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:47.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:47.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:47.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:47.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:47.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:47.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:48.142 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.142 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:48.142 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.142 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:48.143 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:48.144 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:47 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:48.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:47 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:49.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:49.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:49.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:48 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:49.200 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:49.201 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.202 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.203 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.205 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:49.206 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:49.207 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.208 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:42:49.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:49.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:49.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:48 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:49.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:49.417+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:49.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:49.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:49.613 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:49.614 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:49.615 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:50.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:50.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:49 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:50.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:42:50.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:49 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:42:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:51.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:51.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:51 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:51.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:51.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:51.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:51.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:51 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:52 smithi070 ceph-mon[12201]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:52.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:52 smithi070 ceph-mon[12201]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:52.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:52 smithi070 ceph-mon[17138]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:52.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:52 smithi070 ceph-mon[17138]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:52.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:52 smithi102 ceph-mon[13158]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:52.644 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:52 smithi102 ceph-mon[13158]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-08-31T02:42:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:53 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:53.683 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:53.683 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:42:54.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:54 smithi102 ceph-mon[13158]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:42:54.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.462 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[12201]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:42:54.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[17138]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-08-31T02:42:54.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:54.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:42:55.021+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:55.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:55.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:55.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:55.206 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:55.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:42:55.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:55.521 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:42:55.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:42:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:42:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:55 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:42:55.820 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:42:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:42:55] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.33.4" 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:56 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:56 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.673 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.674 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:56.675 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.676 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.677 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:56 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.678 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.679 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:42:56.680 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:42:56.681 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:42:56.682 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:57.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:42:58.375 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:58 smithi070 systemd[1]: Stopping Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:58.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:42:58 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: Upgrade: Updating node-exporter.a (1/2) 2024-08-31T02:42:58.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:42:58 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.a on smithi070 2024-08-31T02:42:59.113 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:58 smithi070 podman[83933]: 2024-08-31 02:42:58.84686161 +0000 UTC m=+0.136703068 container died ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:59.449 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 podman[83933]: 2024-08-31 02:42:59.211252237 +0000 UTC m=+0.501093686 container cleanup ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:59.449 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 bash[83933]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-31T02:42:59.855 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 podman[83961]: 2024-08-31 02:42:59.49601478 +0000 UTC m=+0.649293489 container remove ac9a6992dd5a72d0452587bb1e4a3c5a9331de0ad12a90b5bb3c540b7f596ab1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:42:59.856 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:43:00.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-31T02:43:00.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 systemd[1]: Stopped Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:00.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:42:59 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 1.750s CPU time. 2024-08-31T02:43:00.199 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:00 smithi070 systemd[1]: Starting Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:00 smithi102 ceph-mon[13158]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 799 B/s rd, 0 op/s 2024-08-31T02:43:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:00 smithi070 ceph-mon[12201]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 799 B/s rd, 0 op/s 2024-08-31T02:43:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:00.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:00 smithi070 ceph-mon[17138]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 799 B/s rd, 0 op/s 2024-08-31T02:43:00.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:01.167 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 podman[84208]: 2024-08-31 02:43:00.922393017 +0000 UTC m=+0.017826160 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-31T02:43:01.168 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 podman[84208]: 2024-08-31 02:43:01.046978568 +0000 UTC m=+0.142411708 container create 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 podman[84208]: 2024-08-31 02:43:01.266216071 +0000 UTC m=+0.361649210 container init 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 podman[84208]: 2024-08-31 02:43:01.269093971 +0000 UTC m=+0.364527102 container start 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.272Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.272Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.272Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.272Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-31T02:43:01.645 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=arp 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=edac 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-31T02:43:01.646 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-31T02:43:01.647 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=os 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=stat 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-31T02:43:01.648 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=time 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=uname 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.273Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.274Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a[84280]: ts=2024-08-31T02:43:01.274Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 bash[84208]: 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 2024-08-31T02:43:01.649 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:43:01 smithi070 systemd[1]: Started Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:02.104 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:43:02.104 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.104 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.104 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.104 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.105 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.105 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.105 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:43:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:02.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:02.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:03.094 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:02 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:03.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:02 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:03.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:02 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:03.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:03 smithi102 systemd[1]: Stopping Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:03.825 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:03 smithi102 podman[68235]: 2024-08-31 02:43:03.535119638 +0000 UTC m=+0.120162474 container died 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:04.167 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:03 smithi102 podman[68235]: 2024-08-31 02:43:03.924097817 +0000 UTC m=+0.509140654 container cleanup 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:04.167 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:03 smithi102 bash[68235]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-31T02:43:04.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:03 smithi102 ceph-mon[13158]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:43:04.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:03 smithi102 ceph-mon[13158]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:43:04.167 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:03 smithi102 ceph-mon[13158]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-08-31T02:43:04.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[12201]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:43:04.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[12201]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:43:04.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[12201]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-08-31T02:43:04.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[17138]: Upgrade: Updating node-exporter.b (2/2) 2024-08-31T02:43:04.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[17138]: Deploying daemon node-exporter.b on smithi102 2024-08-31T02:43:04.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:03 smithi070 ceph-mon[17138]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 818 B/s rd, 0 op/s 2024-08-31T02:43:04.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 podman[68247]: 2024-08-31 02:43:04.166757398 +0000 UTC m=+0.635742704 container remove 1b00525ec7eb1b4f9a6d7a14fa4e9146620d3a7bcd17982e203a712bb2017ca1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:04.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:43:04.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-31T02:43:04.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 systemd[1]: Stopped Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:04.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 1.699s CPU time. 2024-08-31T02:43:04.831 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 systemd[1]: Starting Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:04.832 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 podman[68374]: 2024-08-31 02:43:04.5926444 +0000 UTC m=+0.015404500 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-31T02:43:04.832 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 podman[68374]: 2024-08-31 02:43:04.701392179 +0000 UTC m=+0.124152281 container create f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:05.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 podman[68374]: 2024-08-31 02:43:04.954501609 +0000 UTC m=+0.377261710 container init f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:05.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 podman[68374]: 2024-08-31 02:43:04.957529403 +0000 UTC m=+0.380289509 container start f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:43:05.418 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.961Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.961Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=arp 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-31T02:43:05.419 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=edac 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-31T02:43:05.420 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=os 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-31T02:43:05.421 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=stat 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=time 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=uname 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-31T02:43:05.422 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.962Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-31T02:43:05.423 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.963Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-31T02:43:05.423 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b[68389]: ts=2024-08-31T02:43:04.963Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-31T02:43:05.423 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:05 smithi102 bash[68374]: f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 2024-08-31T02:43:05.423 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:43:05 smithi102 systemd[1]: Started Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:05.886 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:43:05] "GET /metrics HTTP/1.1" 200 37981 "" "Prometheus/2.33.4" 2024-08-31T02:43:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-08-31T02:43:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:06.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:06 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:06.392 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 620 B/s rd, 0 op/s 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 798 B/s rd, 0 op/s 2024-08-31T02:43:06.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:06.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:06 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 0s ago 28m - - 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 10s ago 28m - - 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (15m) 0s ago 23m 23.6M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (14m) 10s ago 23m 53.9M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 0s ago 22m 44.9M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (49s) 10s ago 27m 442M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:43:07.075 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (12m) 0s ago 29m 510M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (29m) 0s ago 29m 90.9M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (28m) 10s ago 28m 71.7M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (28m) 0s ago 28m 69.4M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6s) 0s ago 23m 3431k - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 starting - - - - 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (27m) 0s ago 27m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (27m) 0s ago 27m 58.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (26m) 0s ago 26m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (26m) 0s ago 26m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (26m) 10s ago 26m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (25m) 10s ago 25m 51.5M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:43:07.076 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (25m) 10s ago 25m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:43:07.077 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (24m) 10s ago 24m 57.9M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:43:07.077 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (113s) 10s ago 24m 64.8M - 2.33.4 514e6a882f6e c953e8b1c53a 2024-08-31T02:43:07.077 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (22m) 0s ago 22m 78.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:43:07.077 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (22m) 10s ago 22m 77.7M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:07.511 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:43:07.512 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[12201]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-mon[17138]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 0 op/s 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:07 smithi102 ceph-mon[13158]: from='client.15582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:07.894 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading node-exporter daemons", 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:43:07.895 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:07.601+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:07.896 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:07.897 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:07.898 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:07.899 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:07.900 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3497726691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3497726691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:08.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='client.15588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='client.15594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3497726691' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:08.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:08.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:08.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:09.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:09.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:09.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:09.528 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:09.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:09 smithi102 ceph-mon[13158]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:09.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:09.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[12201]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:43:09.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-mon[17138]: from='client.15606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:09.351+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:09.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:09.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:09.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:09.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:09.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:09.718 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:09.718 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:09.718 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:09.718 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:09.718 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:09.719 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:09.720 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.720 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:09.720 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:09.720 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:09.720 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:09.721 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:09.722 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:09.722 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:09.722 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:09.722 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:09 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:10.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.453 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:10.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:10.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:10.456 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:10.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:10.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:10.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:10.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:10.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:10.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:10.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:10.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:10.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:10.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:10.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:10 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:10.962+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:11.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:11.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:11.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:11.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:11.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:11.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:11.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:11.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:10 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:11.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:11.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:11.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:11.695 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:11.696 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:11.697 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:12.391 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:12.393 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:12.394 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.395 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:12.396 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.397 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.398 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:12.399 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.400 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.401 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:12.402 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:12.403 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:12.404 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:12.405 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.406 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:12.407 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:12 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:12.481+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.701 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.702 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:12.703 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:12.704 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:12.705 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:12.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:13.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:13.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:13 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:13.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:13 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.564 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.565 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:14.566 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.567 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.568 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:14.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:14.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:14 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:14.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:14.401+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.708 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:14.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:14.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:14.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:14.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:15.551 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:15.552 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:15.553 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:15.554 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.555 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:15.556 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:15.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:15.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:15.560 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.560 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:15.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:15.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:15.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:15.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:15.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:43:15] "GET /metrics HTTP/1.1" 200 37995 "" "Prometheus/2.33.4" 2024-08-31T02:43:16.445 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:16 smithi102 ceph-mon[13158]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.446 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:16 smithi102 ceph-mon[13158]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.446 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:16 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:16.446 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:16 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:16.446 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:16 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[12201]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[12201]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[17138]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[17138]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1014 B/s rd, 0 op/s 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:16.509+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:16.686 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:16.687 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:16.688 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:16.689 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:16.690 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:16.691 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:16.691 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:16.691 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:16.691 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:16 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:17.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:18.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:18.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:43:18.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:18.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:18.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:18.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:18 smithi102 ceph-mon[13158]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.650 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.651 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:43:18.652 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.653 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:18.654 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:18.655 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.656 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:43:18.657 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:18.658 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:18.659 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:18.660 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[12201]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:18.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:18 smithi070 ceph-mon[17138]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:19.847+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:20.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:20.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:20.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:20.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:20.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:20.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:20.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:19 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:20 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:20.536 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:20 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:21 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:21.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:21.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:21.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:21.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:21.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:22 smithi102 ceph-mon[13158]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:22.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:22 smithi102 ceph-mon[13158]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:22.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[12201]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[12201]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[17138]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[17138]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: Upgrade: Updating prometheus.a 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: Upgrade: Updating prometheus.a 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: Upgrade: Updating prometheus.a 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:23.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:24 smithi102 ceph-mon[13158]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:43:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:24 smithi102 ceph-mon[13158]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:24 smithi070 ceph-mon[12201]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:43:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:24 smithi070 ceph-mon[12201]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:24.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:24 smithi070 ceph-mon[17138]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 880 B/s rd, 0 op/s 2024-08-31T02:43:24.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:24 smithi070 ceph-mon[17138]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:25.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:43:25] "GET /metrics HTTP/1.1" 200 37995 "" "Prometheus/2.33.4" 2024-08-31T02:43:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:26 smithi102 ceph-mon[13158]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:26.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:26 smithi070 ceph-mon[12201]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:26.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:26 smithi070 ceph-mon[17138]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:28.442 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:28 smithi102 ceph-mon[13158]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:43:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:28 smithi070 ceph-mon[12201]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:43:28.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:28 smithi070 ceph-mon[17138]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:43:30.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:30 smithi102 ceph-mon[13158]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-08-31T02:43:30.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:30 smithi070 ceph-mon[17138]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-08-31T02:43:30.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:30 smithi070 ceph-mon[12201]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 721 B/s rd, 0 op/s 2024-08-31T02:43:30.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:32.021 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:31 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.020Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:43:32.345 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[58710]: ts=2024-08-31T02:43:32.021Z caller=main.go:1066 level=info msg="See you next time!" 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 podman[70380]: 2024-08-31 02:43:32.030945125 +0000 UTC m=+0.103993386 container died c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 podman[70380]: 2024-08-31 02:43:32.344985798 +0000 UTC m=+0.418034057 container cleanup c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:32.346 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 bash[70380]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:43:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:32 smithi102 ceph-mon[13158]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:32.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 podman[70392]: 2024-08-31 02:43:32.571275447 +0000 UTC m=+0.541909799 container remove c953e8b1c53a9ac65aa3354f93ae4d23c244d35f83cd0cf0f4b9781af7189ee6 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:32 smithi070 ceph-mon[12201]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:32 smithi070 ceph-mon[17138]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:33.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:43:33.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:33.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:32 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.693s CPU time. 2024-08-31T02:43:33.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:33.468 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:33.468 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 podman[70521]: 2024-08-31 02:43:33.114975927 +0000 UTC m=+0.017067717 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-31T02:43:33.468 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 podman[70521]: 2024-08-31 02:43:33.239697653 +0000 UTC m=+0.141789431 container create f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:33.468 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 podman[70521]: 2024-08-31 02:43:33.441768682 +0000 UTC m=+0.343860474 container init f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:33.468 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 podman[70521]: 2024-08-31 02:43:33.444776324 +0000 UTC m=+0.346868099 container start f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:33.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.467Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-31T02:43:33.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.468Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-31T02:43:33.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.468Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.468Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.468Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.470Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.470Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.471Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.471Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.473Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.473Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.476µs 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.473Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.473Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.494Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.502Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-08-31T02:43:33.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.505Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.510Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.512Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.512Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.512Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.287µs wal_replay_duration=39.843233ms wbl_replay_duration=139ns total_replay_duration=39.883761ms 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.516Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.516Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.516Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 bash[70521]: f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.534Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.971217ms db_storage=947ns remote_storage=1.09µs web_handler=410ns query_engine=627ns scrape=399.992µs scrape_sd=90.946µs notify=10.988µs notify_sd=17.418µs rules=17.133303ms tracing=4.99µs 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.534Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:33.534Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-31T02:43:33.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:33 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:34.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:43:34.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.661 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:34.662 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:34.663 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.663 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:34.663 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:35 smithi102 ceph-mon[13158]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-08-31T02:43:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:35 smithi102 ceph-mon[13158]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:35 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:35 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:35 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[12201]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[12201]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:35.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[17138]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-08-31T02:43:35.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[17138]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:35.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:35.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:35.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:35.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:35.780+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:35.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:35.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:35.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:35.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:35.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:36.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:36.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:36.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:36.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:36.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:37.612 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:37.613 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:37.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:37.617 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.618 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:37.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:37.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:37.622 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:37 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:37.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:37.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 982 B/s rd, 0 op/s 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:37.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:37 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:38.259 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:43:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:38 smithi102 ceph-mon[13158]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:43:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:38 smithi102 ceph-mon[13158]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:38 smithi070 ceph-mon[12201]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:43:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:38 smithi070 ceph-mon[12201]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:38 smithi070 ceph-mon[17138]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:43:38.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:38 smithi070 ceph-mon[17138]: from='client.15612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:38.924 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:43:38.924 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 18s ago 29m - - 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 16s ago 29m - - 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (15m) 18s ago 23m 23.9M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (15m) 16s ago 23m 54.3M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 18s ago 22m 49.2M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (81s) 16s ago 28m 442M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (12m) 18s ago 30m 511M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (30m) 18s ago 30m 93.0M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (28m) 16s ago 28m 70.2M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (28m) 18s ago 28m 68.6M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (37s) 18s ago 24m 7302k - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (34s) 16s ago 24m 7768k - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (28m) 18s ago 28m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:43:38.925 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (27m) 18s ago 27m 58.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (27m) 18s ago 27m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (26m) 18s ago 26m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (26m) 16s ago 26m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (26m) 16s ago 26m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (25m) 16s ago 25m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (25m) 16s ago 25m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 starting - - - - 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (22m) 18s ago 22m 78.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:43:38.926 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (22m) 16s ago 22m 77.7M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:43:39.343 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:43:39.344 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:43:39.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:39.193+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.428 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:39.429 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:39.430 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.431 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.432 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='client.15618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:39.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:39.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:39.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:39 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3632646676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='client.15618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:39.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='client.15618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='client.15624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:39.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:39.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:39.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3632646676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 906 B/s rd, 0 op/s 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:39.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:39.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:39 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3632646676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:39.716 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading prometheus daemons", 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:43:39.717 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:40.763+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:40.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:40.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:40.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:40.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:40.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:40.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:41.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:41.524 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='client.15636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:41.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:41.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:42.621 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-08-31T02:43:42.621 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.621 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.622 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:42.623 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.624 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.625 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.626 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:43:42.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:42.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:42.627 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:42 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-08-31T02:43:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:42.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:42.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:42.357+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:42.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:42.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:42.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:42.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:43.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:43.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:43.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:43.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:43.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:43.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:43.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:43.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:43.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:43 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:43.983+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:44.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:44.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:44.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:44.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:44.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:44.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:44.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:44.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:44.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:44.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:43 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:44 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:44.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:45.495 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:45 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:45.496 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:45.497 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:45.498 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.499 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.500 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:45.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:43:45] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:45 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:45.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:45 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:45.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:45.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:45.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:43:46.650 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:46 smithi102 ceph-mon[13158]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:46 smithi070 ceph-mon[12201]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:46 smithi070 ceph-mon[17138]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: Upgrade: Updating prometheus.a 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.642 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.643 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.643 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-08-31T02:43:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: Upgrade: Updating prometheus.a 2024-08-31T02:43:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: Upgrade: Updating prometheus.a 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:48.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:49.536 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:49 smithi102 ceph-mon[13158]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:49.536 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:49 smithi102 ceph-mon[13158]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:49.536 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:49 smithi102 ceph-mon[13158]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[12201]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[12201]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:49.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[12201]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[17138]: Deploying daemon prometheus.a on smithi102 2024-08-31T02:43:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[17138]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:49 smithi070 ceph-mon[17138]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:51.565 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:51 smithi102 ceph-mon[13158]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:51.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:51 smithi070 ceph-mon[12201]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:51.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:51 smithi070 ceph-mon[17138]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:52.168 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:51 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:52.421 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.170Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.171Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.171Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-31T02:43:52.422 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[70535]: ts=2024-08-31T02:43:52.171Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-31T02:43:52.423 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 podman[72116]: 2024-08-31 02:43:52.180479279 +0000 UTC m=+0.107364618 container died f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:52.720 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 podman[72116]: 2024-08-31 02:43:52.528089891 +0000 UTC m=+0.454975224 container cleanup f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:52.721 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 bash[72116]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:43:53.037 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 podman[72128]: 2024-08-31 02:43:52.720791441 +0000 UTC m=+0.541675788 container remove f88ca5413c034a0c64922bd5749604c99e2bd5e969f7fb632f4468e56353f7c8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:53.037 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:43:53.037 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:53.037 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:52 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 1.106s CPU time. 2024-08-31T02:43:53.344 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:43:53.607 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:53.607 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-mon[13158]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:53.608 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 podman[72255]: 2024-08-31 02:43:53.247656234 +0000 UTC m=+0.016732102 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-31T02:43:53.608 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 podman[72255]: 2024-08-31 02:43:53.363470162 +0000 UTC m=+0.132546030 container create 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:53 smithi070 ceph-mon[12201]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:53.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:43:53.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:53 smithi070 ceph-mon[17138]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 974 B/s rd, 0 op/s 2024-08-31T02:43:53.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 podman[72255]: 2024-08-31 02:43:53.607601967 +0000 UTC m=+0.376677835 container init 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:53.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 podman[72255]: 2024-08-31 02:43:53.610389614 +0000 UTC m=+0.379465493 container start 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:43:53.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.634Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-31T02:43:53.918 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.634Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.634Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.634Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.634Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.636Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.636Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.638Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.638Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.639Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.639Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.724µs 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.639Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.639Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=7 2024-08-31T02:43:53.919 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.658Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.669Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.671Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.673Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.675Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.675Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.676Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=7 maxSegment=7 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.676Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.717µs wal_replay_duration=36.755287ms wbl_replay_duration=105ns total_replay_duration=36.802965ms 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.679Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.679Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.679Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 bash[72255]: 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 2024-08-31T02:43:53.920 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:43:53.921 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.701Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.629912ms db_storage=858ns remote_storage=1.016µs web_handler=435ns query_engine=598ns scrape=377.196µs scrape_sd=94.176µs notify=32.029µs notify_sd=14.807µs rules=20.766961ms tracing=5.77µs 2024-08-31T02:43:53.921 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.701Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-31T02:43:53.921 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:43:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:43:53.701Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-31T02:43:55.046 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:55.047 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.048 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.049 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:55.049 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:55.049 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:55.049 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.049 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:54 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 806 B/s rd, 0 op/s 2024-08-31T02:43:55.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:55.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:55.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:55.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:55.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:55.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:54 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:56.603 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:56.447+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:56.603 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:56.603 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:56.603 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:56.604 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:56.605 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:56.606 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:56.606 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:56.649 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:56.649 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:56.649 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:56.649 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:56.649 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:56.650 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:56.651 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:56.652 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:56.652 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:56.652 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:56.652 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:57.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:57.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:57.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:57.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:57 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:43:57.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.520 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.521 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:58.522 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.523 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.524 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:58.525 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.526 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:58.527 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.528 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.529 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:58.530 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:43:58 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:58.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.577 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.578 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:58.579 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:43:58.580 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:58.581 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:43:58.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:43:58 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:43:59.734+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:59.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:59.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:43:59.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:59.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:43:59.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:43:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:43:59.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:43:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.460 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:00.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:00.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:00.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.461 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:00.527 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 788 B/s rd, 0 op/s 2024-08-31T02:44:00.528 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:00.529 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:01.385 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:01.212+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.386 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:01.387 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:01.388 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:01.389 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.390 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:01.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:01.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:01.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:01.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:01.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:01.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:01.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:01.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 849 B/s rd, 0 op/s 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:01.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:01.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:01.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:02 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:02.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:02.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:02.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:02.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:02.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:02.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:03.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:02.816+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:03.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:03.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:03.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:03.686 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:03.687 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:04 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:04.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:04.434+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:05.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:05.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:05.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:05.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:05] "GET /metrics HTTP/1.1" 200 37992 "" "Prometheus/2.43.0" 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:05.965+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:06.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:06.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.477 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.478 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:06.479 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:06.480 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:06.481 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.482 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.483 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:06.484 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.485 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:06.486 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:06.487 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:06.488 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.582 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.583 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:06.584 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:06.585 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:06.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:06.587 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.665 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.666 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.668 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.670 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.671 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:07.672 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:07.673 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:07.674 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:07.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:07.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:07 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:07.751+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:07.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:07.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:07.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:07.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:07.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:08.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.430 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:08.653 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:08.654 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:09 smithi102 ceph-mon[13158]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:09 smithi102 ceph-mon[13158]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:09 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.648 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.649 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:09.650 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.651 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:09.652 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.653 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[12201]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[12201]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:09.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[17138]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[17138]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:09 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:09.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:09.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:09.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:09.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:10.074 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:44:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:10 smithi102 ceph-mon[13158]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:10.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:10 smithi102 ceph-mon[13158]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:10 smithi070 ceph-mon[12201]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:10.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:10 smithi070 ceph-mon[12201]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:10 smithi070 ceph-mon[17138]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:10.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:10 smithi070 ceph-mon[17138]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:10.737 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:44:10.737 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 9s ago 29m - - 2024-08-31T02:44:10.737 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 3s ago 29m - - 2024-08-31T02:44:10.737 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (16m) 9s ago 24m 24.4M - 0.23.0 ba2b418f427c ea38488406e6 2024-08-31T02:44:10.737 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (15m) 3s ago 24m 54.3M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 9s ago 23m 51.4M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (113s) 3s ago 28m 442M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (13m) 9s ago 30m 514M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (30m) 9s ago 30m 94.5M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (29m) 3s ago 29m 70.1M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (29m) 9s ago 29m 68.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (69s) 9s ago 25m 7684k - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (66s) 3s ago 24m 15.8M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (28m) 9s ago 28m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (28m) 9s ago 28m 58.8M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:44:10.738 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (27m) 9s ago 27m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (27m) 9s ago 27m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (27m) 3s ago 27m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (26m) 3s ago 26m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (26m) 3s ago 26m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (25m) 3s ago 25m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (17s) 3s ago 25m 47.2M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (23m) 9s ago 23m 78.8M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:44:10.739 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (23m) 3s ago 23m 77.9M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:11.186 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:44:11.187 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:44:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[12201]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[12201]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[12201]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/664469543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[17138]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[17138]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[17138]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:11 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/664469543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading alertmanager daemons", 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:44:11.582 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:44:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:11 smithi102 ceph-mon[13158]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:44:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:11 smithi102 ceph-mon[13158]: from='client.15642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:11 smithi102 ceph-mon[13158]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:11.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:11 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/664469543' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:12 smithi102 ceph-mon[13158]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:12 smithi070 ceph-mon[12201]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:12 smithi070 ceph-mon[17138]: from='client.25564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:13 smithi102 ceph-mon[13158]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:44:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:13 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:13 smithi102 ceph-mon[13158]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[12201]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:44:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:13.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[12201]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:13.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[17138]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 941 B/s rd, 0 op/s 2024-08-31T02:44:13.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:13.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:13 smithi070 ceph-mon[17138]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:15.660 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:15 smithi070 ceph-mon[12201]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:15.660 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:15.660 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:15] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-08-31T02:44:15.661 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:15 smithi070 ceph-mon[17138]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:15.661 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:15.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:15 smithi102 ceph-mon[13158]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:16.515 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:16 smithi070 ceph-mon[12201]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-08-31T02:44:16.515 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:16 smithi070 ceph-mon[17138]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-08-31T02:44:16.776 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:16 smithi070 systemd[1]: Stopping Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:16.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:16 smithi102 ceph-mon[13158]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-08-31T02:44:17.330 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[52181]: level=info ts=2024-08-31T02:44:17.037Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:44:17.330 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 podman[89969]: 2024-08-31 02:44:17.078788543 +0000 UTC m=+0.135793589 container died ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:17.614 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 podman[89969]: 2024-08-31 02:44:17.413170062 +0000 UTC m=+0.470175106 container cleanup ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:17.614 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 bash[89969]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-31T02:44:17.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 podman[89980]: 2024-08-31 02:44:17.614208264 +0000 UTC m=+0.530037789 container remove ea38488406e6198b44a69435ea703ad9fcb40c0de05ccb4ca2edf766584c4467 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:17.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 podman[89980]: 2024-08-31 02:44:17.72371559 +0000 UTC m=+0.639545115 volume remove 61cec06e566082613cb6422091ff20b052b2e901a2b2ad0b3f78c7f52b271077 2024-08-31T02:44:18.393 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-31T02:44:18.393 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 systemd[1]: Stopped Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:18.393 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:17 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Consumed 1.497s CPU time. 2024-08-31T02:44:18.393 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 systemd[1]: Starting Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:18.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-mon[12201]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:44:18.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-mon[17138]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:44:18.691 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 podman[90126]: 2024-08-31 02:44:18.295366495 +0000 UTC m=+0.015928522 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-31T02:44:18.691 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 podman[90126]: 2024-08-31 02:44:18.416991357 +0000 UTC m=+0.137553391 volume create 30920a1e59701071e0d89ea5dd9b06a282868b0692381526c3bc4c163a7578c9 2024-08-31T02:44:18.691 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 podman[90126]: 2024-08-31 02:44:18.509123085 +0000 UTC m=+0.229685154 container create 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:18.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:18 smithi102 ceph-mon[13158]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 podman[90126]: 2024-08-31 02:44:18.78700622 +0000 UTC m=+0.507568248 container init 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 podman[90126]: 2024-08-31 02:44:18.789721544 +0000 UTC m=+0.510283572 container start 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.801Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.801Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.803Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.804Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.827Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.827Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:44:19.099 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.831Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-31T02:44:19.100 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:18.831Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-31T02:44:19.100 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 bash[90126]: 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 2024-08-31T02:44:19.100 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:18 smithi070 systemd[1]: Started Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:20.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:20.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:19 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 863 B/s rd, 0 op/s 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:20.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:19 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:20.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:20 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:20.804Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000017596s 2024-08-31T02:44:22.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[12201]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:22.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[12201]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:22.134 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[17138]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:22.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[17138]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:22.135 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:22.120+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:22.136 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:22.137 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:22.138 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:22.139 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:22.140 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:22.141 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE Bus STOPPING 2024-08-31T02:44:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:21 smithi102 ceph-mon[13158]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:21 smithi102 ceph-mon[13158]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:21 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:22.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:44:22.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE Bus STOPPED 2024-08-31T02:44:22.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE Bus STARTING 2024-08-31T02:44:22.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE Serving on http://:::9283 2024-08-31T02:44:22.863 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:22] ENGINE Bus STARTED 2024-08-31T02:44:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:23.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.189 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.190 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:23.191 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.192 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.193 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.194 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.195 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:23.196 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:23.197 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:23.198 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi070:9093"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:23.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:23 smithi102 ceph-mon[13158]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:24.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:23 smithi070 ceph-mon[12201]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:24.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:23 smithi070 ceph-mon[17138]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:24 smithi102 ceph-mon[13158]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:24 smithi102 ceph-mon[13158]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:24 smithi070 ceph-mon[12201]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:24 smithi070 ceph-mon[12201]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:24 smithi070 ceph-mon[17138]: Upgrade: Updating alertmanager.a 2024-08-31T02:44:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:24 smithi070 ceph-mon[17138]: Deploying daemon alertmanager.a on smithi070 2024-08-31T02:44:25.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:25] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-08-31T02:44:26.660 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:26 smithi070 ceph-mon[12201]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-08-31T02:44:26.660 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:26 smithi070 ceph-mon[17138]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-08-31T02:44:26.667 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:26 smithi102 ceph-mon[13158]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 688 B/s rd, 0 op/s 2024-08-31T02:44:27.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 systemd[1]: Stopping Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:27.591 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:27.298Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:44:27.591 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[90140]: ts=2024-08-31T02:44:27.298Z caller=cluster.go:690 level=info component=cluster msg="gossip not settled but continuing anyway" polls=4 elapsed=8.493697017s 2024-08-31T02:44:27.591 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 podman[90885]: 2024-08-31 02:44:27.319069727 +0000 UTC m=+0.123849266 container died 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:27.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 podman[90885]: 2024-08-31 02:44:27.748597983 +0000 UTC m=+0.553377566 container cleanup 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:27.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 bash[90885]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-31T02:44:28.308 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:27 smithi070 podman[90896]: 2024-08-31 02:44:27.95828706 +0000 UTC m=+0.638822809 container remove 2b8ddec4ab33ef703a1abc2fea5dd77a05d84dcc6405f6260fd8adeaac9e4525 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:28.308 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 podman[90896]: 2024-08-31 02:44:28.033883593 +0000 UTC m=+0.714419331 volume remove 30920a1e59701071e0d89ea5dd9b06a282868b0692381526c3bc4c163a7578c9 2024-08-31T02:44:28.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:28 smithi102 ceph-mon[13158]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:28.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-31T02:44:28.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 systemd[1]: Stopped Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:28.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 systemd[1]: Starting Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[12201]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[17138]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:28.993 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 podman[91026]: 2024-08-31 02:44:28.64505475 +0000 UTC m=+0.016152564 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-31T02:44:28.993 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 podman[91026]: 2024-08-31 02:44:28.77653306 +0000 UTC m=+0.147630875 volume create 518517785fda942ba49bef618cd182446c950087c08cacdb45c276af4054bfcf 2024-08-31T02:44:28.993 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:28 smithi070 podman[91026]: 2024-08-31 02:44:28.843698551 +0000 UTC m=+0.214796407 container create df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:29.383 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 podman[91026]: 2024-08-31 02:44:29.087477299 +0000 UTC m=+0.458575120 container init df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:29.383 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 podman[91026]: 2024-08-31 02:44:29.090467312 +0000 UTC m=+0.461565124 container start df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:44:29.383 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.101Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.101Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.103Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.70 port=9094 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.104Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.127Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.127Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.130Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:29.131Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 bash[91026]: df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d 2024-08-31T02:44:29.384 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:29 smithi070 systemd[1]: Started Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:29.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:29.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:29.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:29.542 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:29 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:29.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 764 B/s rd, 0 op/s 2024-08-31T02:44:29.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:29.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:29 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:30.577 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:30.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-08-31T02:44:30.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-08-31T02:44:30.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:30.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:30.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 612 B/s rd, 0 op/s 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 805 B/s rd, 0 op/s 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:30.629 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:30.905 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:30.734+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:30.905 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:30.905 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:30.906 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:30.907 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:30.908 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:30.909 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:30.910 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:31.199 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:31.104Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000374388s 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:31.532 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:31.533 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:32.548 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.548 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.549 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:32.550 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.551 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.552 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:32.553 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:32.554 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:32.555 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:32.556 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.557 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:32.558 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:32 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:32 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:33 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:34.292 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:34.109+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:34.292 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:34.293 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:34.294 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:34.295 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:34.296 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:34.297 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:34.549 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:34.550 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 983 B/s rd, 0 op/s 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:34 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:35.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:35.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:35.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:35.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:35.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:35.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:35.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:35.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:44:35.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:35.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:35.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:35.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:35 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:35.887 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:35] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-08-31T02:44:37.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:37 smithi070 ceph-mon[12201]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:44:37.597 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:37 smithi070 ceph-mon[17138]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:44:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:37 smithi102 ceph-mon[13158]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:44:39.406 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:44:39 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:44:39.106Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001566094s 2024-08-31T02:44:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:39 smithi102 ceph-mon[13158]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-08-31T02:44:39.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:39 smithi070 ceph-mon[12201]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-08-31T02:44:39.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:39 smithi070 ceph-mon[17138]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:41.215+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:41.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:41.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:41.456 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:41.457 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:41.458 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 546 B/s rd, 0 op/s 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.663 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.664 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:41 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:41.953 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:44:42.625 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 1s ago 30m - - 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 15s ago 30m - - 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (13s) 1s ago 25m 17.4M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (16m) 15s ago 24m 54.3M - 8.3.5 dad864ee21e9 127f0cca9fa1 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 1s ago 23m 54.8M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (2m) 15s ago 29m 443M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (13m) 1s ago 31m 516M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (31m) 1s ago 31m 96.3M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (29m) 15s ago 29m 70.5M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:44:42.626 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (29m) 1s ago 29m 65.4M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (101s) 1s ago 25m 8480k - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (98s) 15s ago 25m 15.5M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (29m) 1s ago 29m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (28m) 1s ago 28m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (28m) 1s ago 28m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (28m) 1s ago 28m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (27m) 15s ago 27m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (27m) 15s ago 27m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (26m) 15s ago 26m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (26m) 15s ago 26m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (49s) 15s ago 25m 55.9M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (24m) 1s ago 24m 79.0M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:44:42.627 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (23m) 15s ago 23m 78.0M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[12201]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[12201]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[12201]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[12201]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.629 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:42.630 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.631 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.632 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[17138]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[17138]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[17138]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.633 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[17138]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:42 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.634 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:42.635 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.636 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.637 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.638 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.638 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:42 smithi102 ceph-mon[13158]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:42 smithi102 ceph-mon[13158]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:42 smithi102 ceph-mon[13158]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:42 smithi102 ceph-mon[13158]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:42 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:42.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:42.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:42.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:42.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:42.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:43.065 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:44:43.066 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:44:43.067 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:44:43.402 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:44:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:44:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:44:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading grafana daemons", 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:44:43.403 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:44:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:43 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:43 smithi102 ceph-mon[13158]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:43 smithi102 ceph-mon[13158]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:43 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1474608099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[12201]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[12201]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1474608099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[17138]: from='client.15672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[17138]: from='client.25588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:43.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:43 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1474608099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:44 smithi102 ceph-mon[13158]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:44 smithi102 ceph-mon[13158]: Upgrade: Updating grafana.a 2024-08-31T02:44:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:44 smithi102 ceph-mon[13158]: Deploying daemon grafana.a on smithi102 2024-08-31T02:44:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:44 smithi102 ceph-mon[13158]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:44 smithi102 ceph-mon[13158]: from='client.15696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[12201]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[12201]: Upgrade: Updating grafana.a 2024-08-31T02:44:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[12201]: Deploying daemon grafana.a on smithi102 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[12201]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[12201]: from='client.15696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[17138]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[17138]: Upgrade: Updating grafana.a 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[17138]: Deploying daemon grafana.a on smithi102 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[17138]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:44 smithi070 ceph-mon[17138]: from='client.15696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:44:45.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:45.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:45.700 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:45] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-08-31T02:44:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:44:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:46 smithi070 ceph-mon[12201]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:46.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:46 smithi070 ceph-mon[17138]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:46 smithi102 ceph-mon[13158]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:48.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[17138]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[12201]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:48 smithi102 ceph-mon[13158]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-08-31T02:44:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:48 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:49.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:49 smithi070 ceph-mon[17138]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 0 op/s 2024-08-31T02:44:49.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:49 smithi070 ceph-mon[17138]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-08-31T02:44:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:49 smithi070 ceph-mon[12201]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 0 op/s 2024-08-31T02:44:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:49 smithi070 ceph-mon[12201]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-08-31T02:44:49.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:49 smithi102 ceph-mon[13158]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 732 B/s rd, 0 op/s 2024-08-31T02:44:49.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:49 smithi102 ceph-mon[13158]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 755 B/s rd, 0 op/s 2024-08-31T02:44:51.596 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:51 smithi070 ceph-mon[12201]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-08-31T02:44:51.596 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:51 smithi070 ceph-mon[17138]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-08-31T02:44:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:51 smithi102 ceph-mon[13158]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 583 B/s rd, 0 op/s 2024-08-31T02:44:53.546 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:53 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:53.547 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:53 smithi102 ceph-mon[13158]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:53 smithi070 ceph-mon[12201]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:44:53.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:53 smithi070 ceph-mon[17138]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:53.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:53 smithi102 systemd[1]: Stopping Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:54.261 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[42946]: t=2024-08-31T02:44:53+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-08-31T02:44:54.261 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:53 smithi102 podman[74704]: 2024-08-31 02:44:53.978256904 +0000 UTC m=+0.113817766 container died 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, release=236.1648460182, version=8.5, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.buildah.version=1.24.2, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, architecture=x86_64) 2024-08-31T02:44:54.636 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 podman[74704]: 2024-08-31 02:44:54.359809334 +0000 UTC m=+0.495370195 container cleanup 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, release=236.1648460182, io.buildah.version=1.24.2, name=ubi8, build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.expose-services=) 2024-08-31T02:44:54.636 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 bash[74704]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a 2024-08-31T02:44:54.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 podman[74717]: 2024-08-31 02:44:54.635933609 +0000 UTC m=+0.658869403 container remove 127f0cca9fa1688d13453240b7bee2c743afdf1711572402c01b5e201838ecb0 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.buildah.version=1.24.2, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-type=git, version=8.5, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc.) 2024-08-31T02:44:54.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-31T02:44:54.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 systemd[1]: Stopped Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:54.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 5.876s CPU time. 2024-08-31T02:44:55.319 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:54 smithi102 systemd[1]: Starting Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:44:55.319 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 podman[74842]: 2024-08-31 02:44:55.08807361 +0000 UTC m=+0.016427710 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-08-31T02:44:55.319 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 podman[74842]: 2024-08-31 02:44:55.211982745 +0000 UTC m=+0.140336902 container create 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:44:55.646 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-mon[13158]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 podman[74842]: 2024-08-31 02:44:55.414120413 +0000 UTC m=+0.342474504 container init 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 podman[74842]: 2024-08-31 02:44:55.419239109 +0000 UTC m=+0.347593202 container start 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 bash[74842]: 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 systemd[1]: Started Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.569877871Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-08-31T02:44:55.646 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570049051Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.57005946Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570062809Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570065491Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.57006821Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570070839Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570073858Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570077605Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570080508Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570083101Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570085658Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570088265Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-31T02:44:55.647 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570091325Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.57009792Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570100446Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570103156Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570105678Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=settings t=2024-08-31T02:44:55.570108285Z level=info msg="App mode production" 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore t=2024-08-31T02:44:55.570150011Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore t=2024-08-31T02:44:55.570160445Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-31T02:44:55.648 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.582431047Z level=info msg="Starting DB migrations" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.586065868Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.597571228Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.59883327Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.599209366Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.60047833Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.601613271Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.602697921Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.603038019Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.603538895Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.604141028Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.604862614Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.605984851Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.606968321Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-31T02:44:55.649 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.608183098Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.609287618Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.610359811Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.611235517Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.612663425Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.613948579Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.614832603Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.615988644Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.617127811Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.618041767Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.618881437Z level=info msg="Executing migration" id="create alert_image table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.61977372Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.620890731Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.621274169Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.622116376Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-31T02:44:55.650 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.622972121Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.623064364Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.62383075Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.624714383Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.625561728Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.626415119Z level=info msg="Executing migration" id="create secrets table" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.627256196Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.634124888Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.636909496Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.63980464Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-31T02:44:55.651 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.646092198Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-31T02:44:55.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:55 smithi070 ceph-mon[12201]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:55.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:44:55] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-08-31T02:44:55.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:55 smithi070 ceph-mon[17138]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:44:55.911 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.652655447Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-31T02:44:55.911 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.65552852Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.657974322Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.660734315Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.663455551Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.666249751Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.669003458Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.67182945Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.674637632Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.67745821Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.680291343Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.683178628Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-31T02:44:55.912 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.685926434Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.688702125Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.691583612Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.694041491Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.696868068Z level=info msg="Executing migration" id="add correlation config column" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.699462817Z level=info msg="Executing migration" id="create entity_events table" 2024-08-31T02:44:55.913 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.702057307Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.704799638Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.704898621Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.707519317Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.707611217Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.710271632Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.712996524Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.715593289Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.718342222Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.720968018Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-31T02:44:55.914 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.723704718Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.726356497Z level=info msg="Executing migration" id="Drop public config table" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.729139693Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.731776141Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.734502525Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.737081741Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.737941691Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.741698549Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.742877293Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.744031722Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.744855798Z level=info msg="Executing migration" id="add share column" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.746042005Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.746857659Z level=info msg="Executing migration" id="create file table" 2024-08-31T02:44:55.915 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.747858676Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.748547166Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.749397438Z level=info msg="Executing migration" id="create file_meta table" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.750474295Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.751104951Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.751879125Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.753052521Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.753892011Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.754749229Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.755926814Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.756786338Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.757774891Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-31T02:44:55.916 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.758471717Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.75932643Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.760295715Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.761666397Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.762888781Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.763696885Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.776727116Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.77785119Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.778451424Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.783190601Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.784022613Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-31T02:44:55.917 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.784887459Z level=info msg="Executing migration" id="create folder table" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.78573051Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.786596162Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.787548533Z level=info msg="Executing migration" id="Update folder title length" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.78839305Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=migrator t=2024-08-31T02:44:55.791073831Z level=info msg="migrations completed" performed=105 skipped=376 duration=205.050288ms 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore t=2024-08-31T02:44:55.791401973Z level=info msg="Created default organization" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=plugin.finder t=2024-08-31T02:44:55.823935604Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=secrets t=2024-08-31T02:44:55.824038242Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=query_data t=2024-08-31T02:44:55.82788714Z level=info msg="Query Service initialization" 2024-08-31T02:44:55.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=live.push_http t=2024-08-31T02:44:55.83043255Z level=info msg="Live Push Gateway initialization" 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:56.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:56.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:56.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:56.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:56.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:56 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:56.721 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:56.722 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:56.722 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=infra.usagestats.collector t=2024-08-31T02:44:56.450827602Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-31T02:44:56.722 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=provisioning.datasources t=2024-08-31T02:44:56.452643797Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-08-31T02:44:56.722 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=provisioning.datasources t=2024-08-31T02:44:56.45265433Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-31T02:44:56.722 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=provisioning.datasources t=2024-08-31T02:44:56.452767744Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=provisioning.alerting t=2024-08-31T02:44:56.474867431Z level=info msg="starting to provision alerting" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=provisioning.alerting t=2024-08-31T02:44:56.474877663Z level=info msg="finished to provision alerting" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=grafanaStorageLogger t=2024-08-31T02:44:56.475003178Z level=info msg="storage starting" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=ngalert.state.manager t=2024-08-31T02:44:56.475168575Z level=info msg="Warming state cache for startup" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=ngalert.state.manager t=2024-08-31T02:44:56.475318482Z level=info msg="State cache has been initialized" states=0 duration=148.805µs 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=ticker t=2024-08-31T02:44:56.475484319Z level=info msg=starting first_tick=2024-08-31T02:45:00Z 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=ngalert.multiorg.alertmanager t=2024-08-31T02:44:56.475605786Z level=info msg="starting MultiOrg Alertmanager" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=http.server t=2024-08-31T02:44:56.478838733Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore.transactions t=2024-08-31T02:44:56.497318492Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore.transactions t=2024-08-31T02:44:56.508138867Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-08-31T02:44:56.723 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:44:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=sqlstore.transactions t=2024-08-31T02:44:56.518830782Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:44:57.967+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.179 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.180 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:44:58.181 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:44:58.182 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.183 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:57] ENGINE Bus STOPPING 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:58] ENGINE Bus STOPPED 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:58] ENGINE Bus STARTING 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:58] ENGINE Serving on http://:::9283 2024-08-31T02:44:58.184 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:44:58] ENGINE Bus STARTED 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.910 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.911 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:58.912 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.913 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.914 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.915 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.916 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:58.917 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.918 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.919 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.921 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:58.922 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.923 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:44:58 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.924 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:44:58.925 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:44:58.926 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:44:58.927 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.928 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi102.front.sepia.ceph.com:3000"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:44:58.929 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:44:58 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:00.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:00.407 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:00.407 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:01.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[12201]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:45:01.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.606 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[17138]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:01.351+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:01.607 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:01.608 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:01.609 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:01.610 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:01.611 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:01.612 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:01 smithi102 ceph-mon[13158]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 809 B/s rd, 0 op/s 2024-08-31T02:45:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:01 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 934 B/s rd, 0 op/s 2024-08-31T02:45:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:02.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:02.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:02.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:02.971+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:03.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:03.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:03.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:03.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:03.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:03.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:03 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:03.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:03.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:03.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:03.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:03.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:04.509 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.509 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.510 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:04.511 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.512 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.513 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:04.514 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.515 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.516 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:04.517 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.518 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.519 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:04.520 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:04.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:04.659 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:04.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:04 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:04.608+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:04.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:04.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:04.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:05.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:05.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:05 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:05] "GET /metrics HTTP/1.1" 200 37990 "" "Prometheus/2.43.0" 2024-08-31T02:45:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:05.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:05.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:05.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:05.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:05 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.403 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.403 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.404 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:06.405 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.406 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:06.407 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:06.408 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:06.218+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.409 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.410 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.411 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.412 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:06.413 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.414 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.415 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:06.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:06.417 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:06.418 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.419 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.420 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.420 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.420 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.420 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.420 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:06.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:06.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:06.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:06.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:06 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.655 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.656 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:07.657 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.658 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:07.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.659 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:07.660 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:07 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:07.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:07.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:07.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:07.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:07.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:07.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:07.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:07.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:07.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:07.834+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:08.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:08.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:08.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:08.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:08.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:08.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:08.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:08.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:08.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:07 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:08 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.200 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.201 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:09.208 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:08 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: Upgrade: Updating grafana.a 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: Deploying daemon grafana.a on smithi102 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 990 B/s rd, 0 op/s 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.476 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.477 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.477 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.477 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: Upgrade: Updating grafana.a 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: Deploying daemon grafana.a on smithi102 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 990 B/s rd, 0 op/s 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: Upgrade: Updating grafana.a 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: Deploying daemon grafana.a on smithi102 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 990 B/s rd, 0 op/s 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:11.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:12 smithi102 ceph-mon[13158]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-08-31T02:45:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:12 smithi102 ceph-mon[13158]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:12 smithi102 ceph-mon[13158]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:12 smithi102 ceph-mon[13158]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[12201]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[12201]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[12201]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[12201]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[17138]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-08-31T02:45:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[17138]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[17138]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:12 smithi070 ceph-mon[17138]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:13.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:13 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:13.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:13 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:13.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:13 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:13.761 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:45:14.406 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:45:14.406 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 13s ago 30m - - 2024-08-31T02:45:14.406 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 3s ago 30m - - 2024-08-31T02:45:14.406 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (45s) 13s ago 25m 17.4M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:45:14.406 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (19s) 3s ago 25m 76.2M - 9.4.12 2bacad6d85d8 3444ac5e13e4 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 13s ago 24m 54.9M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (2m) 3s ago 30m 443M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (14m) 13s ago 31m 524M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (31m) 13s ago 31m 95.2M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (30m) 3s ago 30m 70.6M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (30m) 13s ago 30m 67.6M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (2m) 13s ago 26m 8510k - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (2m) 3s ago 26m 16.5M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (29m) 13s ago 29m 57.5M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (29m) 13s ago 29m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (28m) 13s ago 28m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (28m) 13s ago 28m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (28m) 3s ago 28m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:45:14.407 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (27m) 3s ago 27m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:45:14.408 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (27m) 3s ago 27m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:45:14.408 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (26m) 3s ago 26m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:45:14.408 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (81s) 3s ago 26m 63.3M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:45:14.408 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (24m) 13s ago 24m 79.0M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:45:14.408 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (24m) 3s ago 24m 78.1M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:45:14.618 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:14 smithi102 ceph-mon[13158]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:14.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:14 smithi070 ceph-mon[12201]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:14.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:14 smithi070 ceph-mon[17138]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:45:14.799 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:45:14.800 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:45:14.801 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:45:15.156 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:45:15.156 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:45:15.156 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:45:15.156 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-31T02:45:15.156 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr" 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading grafana daemons", 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:45:15.157 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:45:15.166 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:14 smithi102 systemd[1]: Stopping Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:45:15.166 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=server t=2024-08-31T02:45:15.155737917Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-08-31T02:45:15.166 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=tracing t=2024-08-31T02:45:15.156081777Z level=info msg="Closing tracing" 2024-08-31T02:45:15.166 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[74858]: logger=ticker t=2024-08-31T02:45:15.156101254Z level=info msg=stopped last_tick=2024-08-31T02:45:10Z 2024-08-31T02:45:15.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 podman[76590]: 2024-08-31 02:45:15.166033455 +0000 UTC m=+0.115611311 container died 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:15.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-mon[13158]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-mon[13158]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-mon[13158]: from='client.15714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/293997741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:15.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[12201]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[12201]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[12201]: from='client.15714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/293997741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:15.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[17138]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[17138]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[17138]: from='client.15714 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/293997741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:15.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:15.799 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 podman[76590]: 2024-08-31 02:45:15.55588848 +0000 UTC m=+0.505466339 container cleanup 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:15.799 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 bash[76590]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a 2024-08-31T02:45:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:15] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-08-31T02:45:16.064 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 podman[76602]: 2024-08-31 02:45:15.798846739 +0000 UTC m=+0.633409612 container remove 3444ac5e13e4ade71660cd6baeb810d935e0e9b5d98ac05ae7677d7e8827c1ae (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:16.064 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-31T02:45:16.064 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 systemd[1]: Stopped Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:45:16.064 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:15 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 2.530s CPU time. 2024-08-31T02:45:16.375 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 systemd[1]: Starting Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:45:16.477 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:16 smithi070 ceph-mon[12201]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:16.477 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:16 smithi070 ceph-mon[12201]: from='client.25636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:16.478 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:16 smithi070 ceph-mon[17138]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:16.478 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:16 smithi070 ceph-mon[17138]: from='client.25636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-mon[13158]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-mon[13158]: from='client.25636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:16.668 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 podman[76732]: 2024-08-31 02:45:16.27787824 +0000 UTC m=+0.016283952 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-08-31T02:45:16.668 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 podman[76732]: 2024-08-31 02:45:16.393809262 +0000 UTC m=+0.132214962 container create 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:16.668 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 podman[76732]: 2024-08-31 02:45:16.621331346 +0000 UTC m=+0.359737044 container init 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:16.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 podman[76732]: 2024-08-31 02:45:16.624158663 +0000 UTC m=+0.362564371 container start 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 bash[76732]: 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 systemd[1]: Started Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785079552Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785349399Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785366266Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-31T02:45:16.957 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785372795Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.78537784Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785382667Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785387672Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785390328Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785392962Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785396186Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785398878Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785401591Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785404105Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785406758Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785409235Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785411586Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785414321Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785416796Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-31T02:45:16.958 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=settings t=2024-08-31T02:45:16.785419441Z level=info msg="App mode production" 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=sqlstore t=2024-08-31T02:45:16.785462576Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=sqlstore t=2024-08-31T02:45:16.785473924Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=migrator t=2024-08-31T02:45:16.803155346Z level=info msg="Starting DB migrations" 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=migrator t=2024-08-31T02:45:16.81129388Z level=info msg="migrations completed" performed=0 skipped=481 duration=789.714µs 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=sqlstore t=2024-08-31T02:45:16.811873086Z level=info msg="Created default organization" 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=plugin.finder t=2024-08-31T02:45:16.849203961Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=secrets t=2024-08-31T02:45:16.849289394Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=query_data t=2024-08-31T02:45:16.851431418Z level=info msg="Query Service initialization" 2024-08-31T02:45:16.959 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:16 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=live.push_http t=2024-08-31T02:45:16.855415833Z level=info msg="Live Push Gateway initialization" 2024-08-31T02:45:17.668 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=infra.usagestats.collector t=2024-08-31T02:45:17.47253567Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=provisioning.datasources t=2024-08-31T02:45:17.474548497Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=provisioning.datasources t=2024-08-31T02:45:17.474557588Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=provisioning.datasources t=2024-08-31T02:45:17.474674927Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=provisioning.alerting t=2024-08-31T02:45:17.501986447Z level=info msg="starting to provision alerting" 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=provisioning.alerting t=2024-08-31T02:45:17.502003964Z level=info msg="finished to provision alerting" 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=ngalert.state.manager t=2024-08-31T02:45:17.502068196Z level=info msg="Warming state cache for startup" 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=grafanaStorageLogger t=2024-08-31T02:45:17.50213182Z level=info msg="storage starting" 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=ngalert.state.manager t=2024-08-31T02:45:17.502249418Z level=info msg="State cache has been initialized" states=0 duration=180.761µs 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=ticker t=2024-08-31T02:45:17.50231184Z level=info msg=starting first_tick=2024-08-31T02:45:20Z 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=ngalert.multiorg.alertmanager t=2024-08-31T02:45:17.502368974Z level=info msg="starting MultiOrg Alertmanager" 2024-08-31T02:45:17.669 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=http.server t=2024-08-31T02:45:17.504301436Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.040 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.041 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.042 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:17 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 842 B/s rd, 0 op/s 2024-08-31T02:45:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:18.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:18.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:18.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:17 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:18.852 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:18.852 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:18.853 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:18.668+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:18.854 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:18.855 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:18.856 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:18.857 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:18.858 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:18.859 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:18.860 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:19.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.071 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.072 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:20.073 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.074 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:20.075 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.076 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:20.077 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:20.078 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.079 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:20.080 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:20.081 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:19 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:20.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:20.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:20.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 847 B/s rd, 0 op/s 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:20.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:19 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:21.662+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:21.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:21.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:21.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:21.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:21.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:21.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:21 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:22.411 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.412 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:22.412 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:22.412 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.412 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:22.412 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:22 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:22.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.414 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:22.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:23.256+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.399 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.400 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:23.401 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:23.402 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.403 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.404 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:23.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:23.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:23.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:23.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:23 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:23.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:23.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:23.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:23.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:23.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:23.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:23.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:23.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:24.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:24.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:24 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:24.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:24.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:24.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:24.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:24.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:24.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:24.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:24.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:25.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:24.794+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:25.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:25.574 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:25.637 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:25.638 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:25.638 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:25 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:25.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:25] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.601 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.602 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:26.603 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.604 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.605 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:26.606 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:26 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:26.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:26.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:26.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:26.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:26.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:26.282+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:26.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:26.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:26.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:26.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.665 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.665 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.665 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.666 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:27.667 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:27.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:27 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.699 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:27.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:27.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:27.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:27.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:27.891+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:28.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:28.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:28.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:28.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:28.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:28.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:28.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:28.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:28.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:27 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:28.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:28.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:28.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:28.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:28.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:28.680 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:28 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:28.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:28.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:28.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:28.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:28.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:28.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:29.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:29.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:29.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:29.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:29.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.612 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.613 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:29.614 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.615 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:29.616 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:29.617 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:29.617 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:29 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:29.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:29.655+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:29.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:29.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:29.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:29 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:30.430 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:30 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:31.253+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:31.426 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.427 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.428 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.429 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:31.430 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:31.434 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:31.435 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.438 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.523 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:31.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:45:31.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:31.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:31 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:31.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:31.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:31.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:45:31.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:31.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:31.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:31 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:32.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:32.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:32.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: Upgrade: Finalizing container_image settings 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: Upgrade: Complete! 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: Upgrade: Finalizing container_image settings 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: Upgrade: Complete! 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:32.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:32.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.819 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.820 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.821 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:32.822 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:32.823 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: Upgrade: Finalizing container_image settings 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: Upgrade: Complete! 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.824 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 3 op/s 2024-08-31T02:45:32.825 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:32.825 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:32.825 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.825 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:32.825 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:32 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:32.806+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:33.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:33.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:33.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:33.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:33.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:33.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:33.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:32 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:33.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:33 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:45:33.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:33 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:45:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:34 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:34 smithi102 ceph-mon[13158]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:34 smithi102 ceph-mon[13158]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:34 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:34 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:34.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:34.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[17138]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[17138]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[12201]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[12201]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:45:34.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:45:34.595+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.709 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:34.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:34.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:34.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:35.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:35] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-08-31T02:45:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:36 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:36.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:36 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:36.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:36 smithi102 ceph-mon[13158]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:36.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[12201]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:45:36.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:45:36.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:45:36.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:36.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:36 smithi070 ceph-mon[17138]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:45:38.552 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:38 smithi070 ceph-mon[12201]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-08-31T02:45:38.552 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:38 smithi070 ceph-mon[17138]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-08-31T02:45:38.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:38 smithi102 ceph-mon[13158]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 884 B/s rd, 0 op/s 2024-08-31T02:45:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:40 smithi102 ceph-mon[13158]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-08-31T02:45:40.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:40 smithi070 ceph-mon[12201]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-08-31T02:45:40.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:40 smithi070 ceph-mon[17138]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 715 B/s rd, 0 op/s 2024-08-31T02:45:42.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:42.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.288 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.289 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:42 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:42 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:43.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:43 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:43.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:43 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:43.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:43 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:44.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:44 smithi102 ceph-mon[13158]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:44.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:44 smithi070 ceph-mon[12201]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:44.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:44 smithi070 ceph-mon[17138]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:45.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:45.572 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:45 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:45:45.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:45] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-08-31T02:45:46.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-08-31T02:45:46.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:46 smithi070 ceph-mon[12201]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:46.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:46 smithi070 ceph-mon[17138]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:46.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:46 smithi102 ceph-mon[13158]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:47.637 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:47 smithi070 ceph-mon[12201]: from='client.25642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:47.637 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:47 smithi070 ceph-mon[17138]: from='client.25642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:47 smithi102 ceph-mon[13158]: from='client.25642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:48.438 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:45:48.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:48 smithi102 ceph-mon[13158]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:48.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:48 smithi070 ceph-mon[12201]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:48.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:48 smithi070 ceph-mon[17138]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:45:49.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-08-31T02:45:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:49 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/1374097614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:49.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:49 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/1374097614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:49.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:49 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/1374097614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:50.610 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:50 smithi070 ceph-mon[12201]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:50.610 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:50 smithi070 ceph-mon[17138]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:50 smithi102 ceph-mon[13158]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 629 B/s rd, 0 op/s 2024-08-31T02:45:51.397 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)" 2024-08-31T02:45:51.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:51 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/2893664302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:51.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:51 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/2893664302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:51.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:51 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/2893664302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:52.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-08-31T02:45:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:52 smithi102 ceph-mon[13158]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:45:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:52 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:52 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:52 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[12201]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[17138]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:52.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:45:53.499 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:53.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[12201]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-08-31T02:45:53.500 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[12201]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:53.500 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:53.500 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[17138]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-08-31T02:45:53.500 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:53 smithi070 ceph-mon[17138]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:53 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:45:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:53 smithi102 ceph-mon[13158]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 945 B/s rd, 0 op/s 2024-08-31T02:45:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:53 smithi102 ceph-mon[13158]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:54.276 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:45:54.678 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:54 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/139967062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:54.679 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:54 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/139967062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:54.908 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:54 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/139967062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:45:54.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-08-31T02:45:55.541 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:55 smithi102 ceph-mon[13158]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:55 smithi070 ceph-mon[12201]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:55.573 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:55 smithi070 ceph-mon[17138]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:55.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:45:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:45:55] "GET /metrics HTTP/1.1" 200 37991 "" "Prometheus/2.43.0" 2024-08-31T02:45:57.679 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:57 smithi070 ceph-mon[12201]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:57.679 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:57 smithi070 ceph-mon[17138]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:57.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:57 smithi102 ceph-mon[13158]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:58.638 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:58 smithi070 ceph-mon[12201]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:58.638 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:58 smithi070 ceph-mon[17138]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:58.819 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:58 smithi102 ceph-mon[13158]: from='client.15756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:45:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:45:59 smithi102 ceph-mon[13158]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:59.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:45:59 smithi070 ceph-mon[12201]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:59.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:45:59 smithi070 ceph-mon[17138]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:45:59.706 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:46:00.568 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:00.568 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:00 smithi070 ceph-mon[12201]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-08-31T02:46:00.568 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:00.568 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:00 smithi070 ceph-mon[17138]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-08-31T02:46:00.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade status' 2024-08-31T02:46:00.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:00 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:00.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:00 smithi102 ceph-mon[13158]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[12201]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[12201]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[17138]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[17138]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:02 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:02 smithi102 ceph-mon[13158]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:02 smithi102 ceph-mon[13158]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:02.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:02 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": null, 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": false, 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "", 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": null, 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:46:03.262 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:46:03.564 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:03 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:03.565 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:03 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:03.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:03 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:04.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:46:04.626 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:04 smithi070 ceph-mon[12201]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:04.627 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:04 smithi070 ceph-mon[12201]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:04.627 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:04 smithi070 ceph-mon[17138]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:04.627 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:04 smithi070 ceph-mon[17138]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:04 smithi102 ceph-mon[13158]: from='client.15762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:04.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:04 smithi102 ceph-mon[13158]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:05] "GET /metrics HTTP/1.1" 200 37987 "" "Prometheus/2.43.0" 2024-08-31T02:46:06.247 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:06.248 INFO:teuthology.orchestra.run.smithi070.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:06.248 INFO:teuthology.orchestra.run.smithi070.stdout: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:06.570 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:06 smithi070 ceph-mon[12201]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-08-31T02:46:06.571 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:06 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/437147840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:46:06.571 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:06 smithi070 ceph-mon[17138]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-08-31T02:46:06.571 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:06 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/437147840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:46:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:06 smithi102 ceph-mon[13158]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 550 B/s rd, 0 op/s 2024-08-31T02:46:06.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:06 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/437147840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:46:07.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-08-31T02:46:07.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:07 smithi070 ceph-mon[12201]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:07.667 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:07 smithi070 ceph-mon[17138]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:07 smithi102 ceph-mon[13158]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:09 smithi102 ceph-mon[13158]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:09 smithi102 ceph-mon[13158]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:09.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:09 smithi070 ceph-mon[12201]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:09.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:09 smithi070 ceph-mon[12201]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:09 smithi070 ceph-mon[17138]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:09.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:09 smithi070 ceph-mon[17138]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:11.355 INFO:teuthology.orchestra.run.smithi070.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='client.25690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi102", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:11.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='client.25690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi102", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:11.669 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:11 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='client.25690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi102", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:11.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:11 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-31T02:46:12.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:12 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:12.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:12.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:12 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:13.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[12201]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[12201]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[17138]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:13.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:13 smithi070 ceph-mon[17138]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:13 smithi102 ceph-mon[13158]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:13 smithi102 ceph-mon[13158]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:13 smithi102 ceph-mon[13158]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:13 smithi102 ceph-mon[13158]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:13.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:13 smithi102 ceph-mon[13158]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:14.579 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:14.370+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:14.579 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:14.579 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:14.579 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:14.580 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:14.581 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:14.582 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:14.583 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:14.584 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.259 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:46:15.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[12201]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-08-31T02:46:15.572 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:15.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[17138]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:15.577 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.578 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.579 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[12201]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:15.580 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.581 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.582 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[17138]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:15.583 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:15 smithi102 ceph-mon[13158]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 749 B/s rd, 0 op/s 2024-08-31T02:46:15.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:15 smithi102 ceph-mon[13158]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:15.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:15.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:15.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:15 smithi102 ceph-mon[13158]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:46:15.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:15 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 14s ago 31m - - 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 3s ago 31m - - 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (107s) 14s ago 26m 17.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (59s) 3s ago 26m 79.8M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:46:15.935 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 14s ago 25m 58.1M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (3m) 3s ago 31m 444M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (15m) 14s ago 32m 527M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (32m) 14s ago 32m 99.9M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (31m) 3s ago 31m 73.8M 2048M 17.2.0 e1d6a67b021e 459658b88b6d 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (31m) 14s ago 31m 71.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 14s ago 27m 16.5M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (3m) 3s ago 27m 15.7M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (30m) 14s ago 30m 57.6M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (30m) 14s ago 30m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (29m) 14s ago 29m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (29m) 14s ago 29m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:46:15.936 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (29m) 3s ago 29m 56.8M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (28m) 3s ago 28m 51.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (28m) 3s ago 28m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (28m) 3s ago 27m 58.0M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 3s ago 27m 64.0M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (25m) 14s ago 25m 79.3M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:46:15.937 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (25m) 3s ago 25m 78.3M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:46:15.937 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:15] "GET /metrics HTTP/1.1" 200 37989 "" "Prometheus/2.43.0" 2024-08-31T02:46:16.332 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:46:16.332 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:46:16.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:16.334 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:46:16.334 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-08-31T02:46:16.334 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:46:16.334 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:46:16.334 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:46:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:16 smithi102 ceph-mon[13158]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:16 smithi102 ceph-mon[13158]: from='client.? 172.21.15.70:0/3823894400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:16.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:16 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi102", 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "0/1 daemons upgraded", 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-08-31T02:46:16.674 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:46:16.675 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:46:16.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[12201]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:16.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3823894400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:16.675 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:16.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[17138]: from='client.25696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:16.675 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3823894400' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:16.676 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:16 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='client.15798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:17.667 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all mgr 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[12201]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='client.15798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:17.668 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:17.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all mgr 2024-08-31T02:46:17.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:17.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:17.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:17.669 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:17 smithi070 ceph-mon[17138]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:17.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='client.25702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='client.15798 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: Upgrade: Setting container_image for all mgr 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:17.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:17 smithi102 ceph-mon[13158]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:18.526 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:18 smithi102 ceph-mon[13158]: from='client.15810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:18.526 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:18.526 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:18.526 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:18.526 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:18 smithi102 ceph-mon[13158]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[12201]: from='client.15810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:18.684 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[17138]: from='client.15810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:18.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:18.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:18.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:18.688 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:18 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:19.455 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 ceph-mon[13158]: Upgrade: Updating mon.b 2024-08-31T02:46:19.455 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 ceph-mon[13158]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:19.455 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 ceph-mon[13158]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-08-31T02:46:19.455 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 systemd[1]: Stopping Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:46:19.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[12201]: Upgrade: Updating mon.b 2024-08-31T02:46:19.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[12201]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:19.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[12201]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-08-31T02:46:19.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[17138]: Upgrade: Updating mon.b 2024-08-31T02:46:19.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[17138]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:19.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:19 smithi070 ceph-mon[17138]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 743 B/s rd, 0 op/s 2024-08-31T02:46:19.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[13151]: 2024-08-31T02:46:19.691+0000 7f99795c5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:46:19.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[13151]: 2024-08-31T02:46:19.691+0000 7f99795c5700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-08-31T02:46:20.202 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:19 smithi102 podman[79213]: 2024-08-31 02:46:19.937281987 +0000 UTC m=+0.338617956 container died 459658b88b6d9bf217c710501b7cfd13353be8f26c90d496df8b5991bfef3909 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, version=8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, name=centos-stream, architecture=x86_64, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0) 2024-08-31T02:46:20.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 podman[79213]: 2024-08-31 02:46:20.291781398 +0000 UTC m=+0.693117370 container cleanup 459658b88b6d9bf217c710501b7cfd13353be8f26c90d496df8b5991bfef3909 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, architecture=x86_64, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0) 2024-08-31T02:46:20.585 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 bash[79213]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b 2024-08-31T02:46:20.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 podman[79225]: 2024-08-31 02:46:20.584871795 +0000 UTC m=+0.641882606 container remove 459658b88b6d9bf217c710501b7cfd13353be8f26c90d496df8b5991bfef3909 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, release=754, version=8, vendor=Red Hat, Inc., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, distribution-scope=public, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-08-31T02:46:20.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-31T02:46:20.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 systemd[1]: Stopped Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:46:20.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service: Consumed 20.437s CPU time. 2024-08-31T02:46:20.912 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:20 smithi102 systemd[1]: Starting Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:46:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 podman[79357]: 2024-08-31 02:46:21.111560074 +0000 UTC m=+0.104989225 container create 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True) 2024-08-31T02:46:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 podman[79357]: 2024-08-31 02:46:21.02236026 +0000 UTC m=+0.015789416 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 podman[79357]: 2024-08-31 02:46:21.330745006 +0000 UTC m=+0.324174154 container init 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 podman[79357]: 2024-08-31 02:46:21.334043 +0000 UTC m=+0.327472148 container start 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: set uid:gid to 167:167 (ceph:ceph) 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc), process ceph-mon, pid 2 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: pidfile_write: ignore empty --pid-file 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: load: jerasure load: lrc 2024-08-31T02:46:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: RocksDB version: 7.9.2 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Git sha 0 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Compile date 2024-08-29 18:05:07 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: DB SUMMARY 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: DB Session ID: QAZAWDN74WEHG7RLPXV8 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: CURRENT file: CURRENT 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: IDENTITY file: IDENTITY 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: MANIFEST file: MANIFEST-000009 size: 4220 Bytes 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000075.sst 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000073.log size: 4411695 ; 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.error_if_exists: 0 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.create_if_missing: 0 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.paranoid_checks: 1 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-31T02:46:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.env: 0x56235becb940 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.fs: PosixFileSystem 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.info_log: 0x56235c879560 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_file_opening_threads: 16 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.statistics: (nil) 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.use_fsync: 0 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_log_file_size: 0 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.keep_log_file_num: 1000 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.recycle_log_file_num: 0 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_fallocate: 1 2024-08-31T02:46:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_mmap_reads: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_mmap_writes: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.use_direct_reads: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.create_missing_column_families: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.db_log_dir: 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.wal_dir: 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.advise_random_on_open: 1 2024-08-31T02:46:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.db_write_buffer_size: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.write_buffer_manager: 0x56235c8888c0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.rate_limiter: (nil) 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.wal_recovery_mode: 2 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enable_thread_tracking: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enable_pipelined_write: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.unordered_write: 0 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-31T02:46:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-31T02:46:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-31T02:46:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.row_cache: None 2024-08-31T02:46:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.wal_filter: None 2024-08-31T02:46:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_ingest_behind: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.two_write_queues: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.manual_wal_flush: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.wal_compression: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.atomic_flush: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.log_readahead_size: 0 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-31T02:46:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.best_efforts_recovery: 0 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.allow_data_in_errors: 0 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.db_host_id: __hostname__ 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_background_jobs: 2 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_background_compactions: -1 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_subcompactions: 1 2024-08-31T02:46:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_total_wal_size: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_open_files: -1 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bytes_per_sync: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_readahead_size: 0 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_background_flushes: -1 2024-08-31T02:46:21.677 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Compression algorithms supported: 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kZSTD supported: 0 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kXpressCompression supported: 0 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kBZip2Compression supported: 0 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kLZ4Compression supported: 1 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kZlibCompression supported: 1 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kLZ4HCCompression supported: 1 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: kSnappyCompression supported: 1 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-31T02:46:21.678 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.merge_operator: 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_filter: None 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_filter_factory: None 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.sst_partitioner_factory: None 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56235c879180) 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_type: 0 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: data_block_index_type: 0 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_shortening: 1 2024-08-31T02:46:21.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: checksum: 4 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: no_block_cache: 0 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache: 0x56235c86d090 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_options: 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: capacity : 536870912 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: num_shard_bits : 4 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: strict_capacity_limit : 0 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_compressed: (nil) 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: persistent_cache: (nil) 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_size: 4096 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_size_deviation: 10 2024-08-31T02:46:21.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_restart_interval: 16 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_block_restart_interval: 1 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: metadata_block_size: 4096 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: partition_filters: 0 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: use_delta_encoding: 1 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: filter_policy: bloomfilter 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: whole_key_filtering: 1 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: verify_compression: 0 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: format_version: 5 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: enable_index_compression: 1 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_align: 0 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: max_auto_readahead_size: 262144 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: prepopulate_block_cache: 0 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: initial_auto_readahead_size: 8192 2024-08-31T02:46:21.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: num_file_reads_for_auto_readahead: 2 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.write_buffer_size: 33554432 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_write_buffer_number: 2 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression: NoCompression 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression: Disabled 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.prefix_extractor: nullptr 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.num_levels: 7 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-31T02:46:21.682 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.level: 32767 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.strategy: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.enabled: false 2024-08-31T02:46:21.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.target_file_size_base: 67108864 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-31T02:46:21.684 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.arena_block_size: 1048576 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.disable_auto_compactions: 0 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-31T02:46:21.685 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.inplace_update_support: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.bloom_locality: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.max_successive_merges: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-31T02:46:21.686 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.paranoid_file_checks: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.force_consistency_checks: 1 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.report_bg_io_stats: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.ttl: 2592000 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enable_blob_files: false 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.min_blob_size: 0 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_file_size: 268435456 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-31T02:46:21.687 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.blob_file_starting_level: 0 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 75.sst 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 77, last_sequence is 40287, log_number is 73,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 73 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0a131e42-49fb-4f69-967c-607f5ac729de 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072381360864, "job": 1, "event": "recovery_started", "wal_files": [73]} 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #73 mode 2 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072381371462, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 78, "file_size": 2685353, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 40292, "largest_seqno": 42787, "table_properties": {"data_size": 2674851, "index_size": 6447, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3013, "raw_key_size": 26412, "raw_average_key_size": 22, "raw_value_size": 2651504, "raw_average_value_size": 2224, "num_data_blocks": 284, "num_entries": 1192, "num_filter_entries": 1192, "num_deletions": 258, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725072381, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0a131e42-49fb-4f69-967c-607f5ac729de", "db_session_id": "QAZAWDN74WEHG7RLPXV8", "orig_file_number": 78, "seqno_to_time_mapping": "N/A"}} 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072381371585, "job": 1, "event": "recovery_finished"} 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:5047] Creating manifest 80 2024-08-31T02:46:21.688 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000073.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56235c89ae00 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: DB pointer 0x56235c996000 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** DB Stats ** 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-31T02:46:21.689 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** Compaction Stats [default] ** 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: L0 1/0 2.56 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 604.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: L6 1/0 11.83 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: Sum 2/0 14.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 604.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 604.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** Compaction Stats [default] ** 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:46:21.690 INFO:journalctl@ceph.mon.b.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 604.0 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative compaction: 0.00 GB write, 172.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval compaction: 0.00 GB write, 172.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:46:21.691 INFO:journalctl@ceph.mon.b.smithi102.stdout: Block cache BinnedLRUCache@0x56235c86d090#2 capacity: 512.00 MB usage: 9.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,3.03 KB,0.000578165%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: starting mon.b rank 2 at public addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] at bind addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???) e3 preinit fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).mds e1 new map 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).mds e1 print_map 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: e1 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: legacy client fscid: -1 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:21.692 INFO:journalctl@ceph.mon.b.smithi102.stdout: No filesystems configured 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 ceph-mon[79375]: mon.b@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 bash[79357]: 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 2024-08-31T02:46:21.693 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:21 smithi102 systemd[1]: Started Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:46:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: mon.b calling monitor election 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: mon.c calling monitor election 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: mon.a calling monitor election 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:22.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:22.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: mon.b calling monitor election 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: mon.c calling monitor election 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:22.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:22 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: mon.b calling monitor election 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: mon.c calling monitor election 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: mon.a calling monitor election 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: fsmap 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:22.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 954 B/s rd, 0 op/s 2024-08-31T02:46:22.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:22.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:22 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:23.291+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:23.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:23.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:46:23.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:23.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:23.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:23.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:23.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:23 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:23.941 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.941 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.941 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.941 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.941 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.942 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:23.943 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:23.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.947 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:23.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:23.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:23.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1005 B/s rd, 0 op/s 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:23.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:23 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:24.992+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:25.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:25.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:25.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:25.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:25.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:25.259 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:25 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:25.833 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:25] "GET /metrics HTTP/1.1" 200 37989 "" "Prometheus/2.43.0" 2024-08-31T02:46:26.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.322 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.322 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.322 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.322 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.323 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:26.324 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.325 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.326 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:26.327 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:26 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:26.384 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.384 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.384 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:26.385 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:26.386 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.387 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.388 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:26.389 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:26.390 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.391 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.392 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.393 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.394 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:26.395 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:26.396 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:26.396 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:26.396 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:26.636 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:26.383+0000 7f8ca349f640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:26.704+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:26.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:26.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:26.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:26.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:26.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:26.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:26.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:27.533 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.533 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:27.533 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:27.534 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:27.535 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:27 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:27.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:27 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.449 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.450 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:28.451 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.452 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.453 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.454 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.455 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:28.456 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.457 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.458 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:28.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:28.259+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.461 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.462 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:28.463 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:28.464 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.465 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.466 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.466 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.466 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.466 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.466 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:28 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:28.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:28.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:28.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: pgmap v335: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: pgmap v336: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 0 op/s 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:28.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:28.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:28.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:28.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:28 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.543 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.544 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:29.545 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.548 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.549 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:29.550 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:29 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:29.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:29.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:29.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:29.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:29.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:29.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:46:29.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:29.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:29.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:29 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:30.026+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:30.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:30.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:30.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:30.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:30.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:30.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:30.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:30.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:30.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:30.652 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:30.653 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:30.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:30 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:31 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:31.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:31.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.688 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:31.690 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:46:31.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:31.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:31.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:31.830+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:31.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:31.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:31.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:31.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:31.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:31.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:31.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:31.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:32 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.689 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.690 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:32.691 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:32 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:33 smithi102 ceph-mon[79375]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:33.700 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:33.701 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:33.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[17138]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.709 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-mon[12201]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:33.528+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:33.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:33.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:33.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:33.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:33 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:34.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:34 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:34.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:34 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:34.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:34 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:34.599 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:34 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:46:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:34 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-31T02:46:35.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:35 smithi102 ceph-mon[79375]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.675 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.676 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:35.677 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:35.678 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.679 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.680 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.681 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.682 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.683 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:35 smithi102 ceph-mon[79375]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:35.683 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:35 smithi102 ceph-mon[79375]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:35.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:35] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:35.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:35.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:35.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[12201]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[12201]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:35.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:35.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:35.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[17138]: Upgrade: It appears safe to stop mon.b 2024-08-31T02:46:35.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:35 smithi070 ceph-mon[17138]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:37.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:46:37.343 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:37.344 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.344 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:37.344 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:37.344 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:37 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:37.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:46:37.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:37.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:37.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:46:37.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:46:37.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:38.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:38 smithi102 ceph-mon[79375]: Upgrade: Updating mon.b 2024-08-31T02:46:38.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:38 smithi102 ceph-mon[79375]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:38.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:38 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:38 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:38 smithi102 ceph-mon[79375]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[17138]: Upgrade: Updating mon.b 2024-08-31T02:46:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[17138]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[12201]: Upgrade: Updating mon.b 2024-08-31T02:46:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[12201]: Deploying daemon mon.b on smithi102 2024-08-31T02:46:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:38.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:38 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:39.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:39 smithi070 ceph-mon[17138]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-08-31T02:46:39.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:39 smithi070 ceph-mon[17138]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:39 smithi070 ceph-mon[12201]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-08-31T02:46:39.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:39 smithi070 ceph-mon[12201]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:39.414 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 ceph-mon[79375]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 912 B/s rd, 0 op/s 2024-08-31T02:46:39.415 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 ceph-mon[79375]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 998 B/s rd, 0 op/s 2024-08-31T02:46:39.415 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 systemd[1]: Stopping Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:46:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[79371]: 2024-08-31T02:46:39.413+0000 7f227a454640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:46:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[79371]: 2024-08-31T02:46:39.413+0000 7f227a454640 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-08-31T02:46:39.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 podman[81037]: 2024-08-31 02:46:39.577132182 +0000 UTC m=+0.304131990 container died 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:46:40.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 podman[81037]: 2024-08-31 02:46:39.955318904 +0000 UTC m=+0.682318722 container cleanup 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0) 2024-08-31T02:46:40.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:39 smithi102 bash[81037]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b 2024-08-31T02:46:40.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 podman[81050]: 2024-08-31 02:46:40.332413546 +0000 UTC m=+0.754809910 container remove 383426ab228225aaf88933c1eaa5a619fbeb1d0ad523db56b925e86828e85ab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-31T02:46:40.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-31T02:46:40.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 systemd[1]: Stopped Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:46:40.586 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service: Consumed 1.063s CPU time. 2024-08-31T02:46:40.910 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 systemd[1]: Starting Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:46:41.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 podman[81178]: 2024-08-31 02:46:40.813030165 +0000 UTC m=+0.016633013 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:46:41.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:40 smithi102 podman[81178]: 2024-08-31 02:46:40.976700864 +0000 UTC m=+0.180303712 container create 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731) 2024-08-31T02:46:41.513 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 podman[81178]: 2024-08-31 02:46:41.195290839 +0000 UTC m=+0.398893695 container init 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 podman[81178]: 2024-08-31 02:46:41.19994624 +0000 UTC m=+0.403549087 container start 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: set uid:gid to 167:167 (ceph:ceph) 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc), process ceph-mon, pid 2 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: pidfile_write: ignore empty --pid-file 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: load: jerasure load: lrc 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: RocksDB version: 7.9.2 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Git sha 0 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Compile date 2024-08-29 18:05:07 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: DB SUMMARY 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: DB Session ID: KHQ8V9TW83MND8AX2IPI 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: CURRENT file: CURRENT 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: IDENTITY file: IDENTITY 2024-08-31T02:46:41.514 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: MANIFEST file: MANIFEST-000080 size: 510 Bytes 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000085.sst 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000083.log size: 619626 ; 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.error_if_exists: 0 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.create_if_missing: 0 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.paranoid_checks: 1 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.env: 0x5618f9367940 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.fs: PosixFileSystem 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.info_log: 0x5618fbbb3560 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_file_opening_threads: 16 2024-08-31T02:46:41.515 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.statistics: (nil) 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.use_fsync: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_log_file_size: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.keep_log_file_num: 1000 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.recycle_log_file_num: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_fallocate: 1 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_mmap_reads: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_mmap_writes: 0 2024-08-31T02:46:41.517 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.use_direct_reads: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.create_missing_column_families: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.db_log_dir: 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.wal_dir: 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.advise_random_on_open: 1 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.db_write_buffer_size: 0 2024-08-31T02:46:41.518 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.write_buffer_manager: 0x5618fbbc28c0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.rate_limiter: (nil) 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.wal_recovery_mode: 2 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enable_thread_tracking: 0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enable_pipelined_write: 0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.unordered_write: 0 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.row_cache: None 2024-08-31T02:46:41.519 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.wal_filter: None 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_ingest_behind: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.two_write_queues: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.manual_wal_flush: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.wal_compression: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.atomic_flush: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.log_readahead_size: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.best_efforts_recovery: 0 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-31T02:46:41.520 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.allow_data_in_errors: 0 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.db_host_id: __hostname__ 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_background_jobs: 2 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_background_compactions: -1 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_subcompactions: 1 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_total_wal_size: 0 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-31T02:46:41.521 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_open_files: -1 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bytes_per_sync: 0 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-31T02:46:41.522 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_readahead_size: 0 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_background_flushes: -1 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Compression algorithms supported: 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kZSTD supported: 0 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kXpressCompression supported: 0 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kBZip2Compression supported: 0 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kLZ4Compression supported: 1 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kZlibCompression supported: 1 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kLZ4HCCompression supported: 1 2024-08-31T02:46:41.523 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: kSnappyCompression supported: 1 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000080 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.merge_operator: 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_filter: None 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_filter_factory: None 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.sst_partitioner_factory: None 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5618fbbb3180) 2024-08-31T02:46:41.524 INFO:journalctl@ceph.mon.b.smithi102.stdout: cache_index_and_filter_blocks: 1 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: pin_top_level_index_and_filter: 1 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_type: 0 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: data_block_index_type: 0 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_shortening: 1 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: checksum: 4 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: no_block_cache: 0 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache: 0x5618fbba7090 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_name: BinnedLRUCache 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_options: 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: capacity : 536870912 2024-08-31T02:46:41.525 INFO:journalctl@ceph.mon.b.smithi102.stdout: num_shard_bits : 4 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: strict_capacity_limit : 0 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: high_pri_pool_ratio: 0.000 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_cache_compressed: (nil) 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: persistent_cache: (nil) 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_size: 4096 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_size_deviation: 10 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_restart_interval: 16 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: index_block_restart_interval: 1 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: metadata_block_size: 4096 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: partition_filters: 0 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: use_delta_encoding: 1 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: filter_policy: bloomfilter 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: whole_key_filtering: 1 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: verify_compression: 0 2024-08-31T02:46:41.526 INFO:journalctl@ceph.mon.b.smithi102.stdout: read_amp_bytes_per_bit: 0 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: format_version: 5 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: enable_index_compression: 1 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: block_align: 0 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: max_auto_readahead_size: 262144 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: prepopulate_block_cache: 0 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: initial_auto_readahead_size: 8192 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout: num_file_reads_for_auto_readahead: 2 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.write_buffer_size: 33554432 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_write_buffer_number: 2 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression: NoCompression 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression: Disabled 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.prefix_extractor: nullptr 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-31T02:46:41.527 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.num_levels: 7 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-31T02:46:41.528 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.level: 32767 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.strategy: 0 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.enabled: false 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.target_file_size_base: 67108864 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-31T02:46:41.529 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.arena_block_size: 1048576 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-31T02:46:41.530 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.disable_auto_compactions: 0 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.inplace_update_support: 0 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-31T02:46:41.531 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.bloom_locality: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.max_successive_merges: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.paranoid_file_checks: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.force_consistency_checks: 1 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.report_bg_io_stats: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.ttl: 2592000 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enable_blob_files: false 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.min_blob_size: 0 2024-08-31T02:46:41.532 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_file_size: 268435456 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.blob_file_starting_level: 0 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000080 succeeded,manifest_file_number is 80, next_file_number is 87, last_sequence is 43364, log_number is 83,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 83 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 83 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0a131e42-49fb-4f69-967c-607f5ac729de 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072401226188, "job": 1, "event": "recovery_started", "wal_files": [83]} 2024-08-31T02:46:41.533 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #83 mode 2 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072401228428, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 88, "file_size": 379573, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 43369, "largest_seqno": 43626, "table_properties": {"data_size": 377260, "index_size": 955, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2986, "raw_average_key_size": 24, "raw_value_size": 374374, "raw_average_value_size": 3019, "num_data_blocks": 42, "num_entries": 124, "num_filter_entries": 124, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725072401, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0a131e42-49fb-4f69-967c-607f5ac729de", "db_session_id": "KHQ8V9TW83MND8AX2IPI", "orig_file_number": 88, "seqno_to_time_mapping": "N/A"}} 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072401228529, "job": 1, "event": "recovery_finished"} 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:5047] Creating manifest 90 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000083.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5618fbbd4e00 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: DB pointer 0x5618fbcd0000 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** DB Stats ** 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:41.534 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** Compaction Stats [default] ** 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: L0 1/0 370.68 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 279.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: L6 1/0 11.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Sum 2/0 12.12 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 279.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 279.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:46:41.535 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** Compaction Stats [default] ** 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 279.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:46:41.536 INFO:journalctl@ceph.mon.b.smithi102.stdout: Cumulative compaction: 0.00 GB write, 81.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: Interval compaction: 0.00 GB write, 81.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: Block cache BinnedLRUCache@0x5618fbba7090#2 capacity: 512.00 MB usage: 1.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: starting mon.b rank 2 at public addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] at bind addrs [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???) e3 preinit fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).mds e1 new map 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).mds e1 print_map 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: e1 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-31T02:46:41.537 INFO:journalctl@ceph.mon.b.smithi102.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout: legacy client fscid: -1 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout: 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout: No filesystems configured 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 ceph-mon[81198]: mon.b@-1(???).paxosservice(auth 1..31) refresh upgraded, format 0 -> 3 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 bash[81178]: 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 2024-08-31T02:46:41.538 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:41 smithi102 systemd[1]: Started Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:46:45.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:45] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-08-31T02:46:47.038 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:46:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: mon.b calling monitor election 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: mon.c calling monitor election 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: mon.a calling monitor election 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: fsmap 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:47.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[12201]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: mon.b calling monitor election 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: mon.c calling monitor election 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-08-31T02:46:47.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:47.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:47.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:47 smithi070 ceph-mon[17138]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 5s ago 32m - - 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 10s ago 32m - - 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 5s ago 27m 27.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (91s) 10s ago 26m 80.8M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 5s ago 25m 58.1M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:46:47.702 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (4m) 10s ago 31m 445M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (15m) 5s ago 33m 527M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (33m) 5s ago 33m 104M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 starting - - - 2048M 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (32m) 5s ago 32m 74.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (3m) 5s ago 27m 16.3M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (3m) 10s ago 27m 15.8M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (31m) 5s ago 31m 57.6M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (30m) 5s ago 30m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (30m) 5s ago 30m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (30m) 5s ago 30m 53.4M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (29m) 10s ago 29m 56.9M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (29m) 10s ago 29m 51.9M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:46:47.703 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (28m) 10s ago 28m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:46:47.704 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (28m) 10s ago 28m 58.1M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:46:47.704 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 10s ago 27m 66.7M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:46:47.704 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (26m) 5s ago 26m 79.4M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:46:47.704 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (26m) 10s ago 26m 78.6M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:46:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: mon.b calling monitor election 2024-08-31T02:46:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: mon.c calling monitor election 2024-08-31T02:46:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: mon.a calling monitor election 2024-08-31T02:46:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:46:47.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: monmap e3: 3 mons at {a=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],b=[v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0],c=[v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0]} 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: fsmap 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: mgrmap e45: y(active, since 5m), standbys: x 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:46:47.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:47 smithi102 ceph-mon[81198]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:46:48.110 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 1 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:48.111 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 3 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:46:48.112 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:46:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:48.161+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.451 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:48.452 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:48.453 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.454 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.455 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.476 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:46:48.476 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:46:48.476 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi102", 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "0/1 daemons upgraded", 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:46:48.477 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:46:48.855 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='client.25738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:48.855 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/949540735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.855 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.856 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.857 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:48.858 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:48.859 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.860 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.861 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.862 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.862 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.862 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.862 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='client.25738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:48.863 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/949540735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.863 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.863 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.863 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.864 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.865 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:48.866 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:48.867 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.868 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.869 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.870 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.871 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:48.872 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:48.873 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.873 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:48 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='client.25738 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/949540735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:48.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:48.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:48.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 762 B/s rd, 0 op/s 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:48.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:48.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:48.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:48 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:49.491 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-mon[12201]: from='client.25750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:49.796 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-mon[17138]: from='client.25750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:49.881 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:49 smithi102 ceph-mon[81198]: from='client.25750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:49.910+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:50.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:50.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:50.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:49 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.733 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:50.734 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.735 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.736 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:50.737 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:50.738 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.739 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:50.740 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.741 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.742 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:50.743 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:50.744 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:50.745 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.745 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:50 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:50.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:50.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:50.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:50.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:50 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:52.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:51.711+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:52.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:52.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:52.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:52.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:52.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:52.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:52.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:52.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:52.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:52.375 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:52 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:52.541 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:52.542 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:52 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:53.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:53.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.674 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:53.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:53.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:53 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:53.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:53.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:53.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:53.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:53.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:53.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:53.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:53.447+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:53.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 969 B/s rd, 0 op/s 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:53.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:53.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:53.716 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:53.717 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:54.346 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:54.347 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:54 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:54.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:54 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:55.047+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:55.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.418 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.419 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:55.420 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.421 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.422 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:55.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:55.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:55.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:55 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.566 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.566 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.566 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.567 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:55.568 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.569 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.570 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:55.571 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.572 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.573 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:55.574 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:55.575 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.576 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 968 B/s rd, 0 op/s 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:55.577 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:55.578 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:55.579 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:55.905 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:46:55] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-08-31T02:46:56.614 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.614 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.615 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.616 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.617 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:56.618 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:56.619 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.619 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.620 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.621 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.622 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:56.623 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:56.624 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:56.624 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:56.625 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:56.626 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.627 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.628 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.629 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.630 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:56.631 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:56.632 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.668 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.669 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:56.670 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.671 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.672 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 996 B/s rd, 0 op/s 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:56.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:56 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:56.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:56.717+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:56.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:56.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:56.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:56.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:56.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:56.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:57.558 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.558 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:57.558 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.558 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:57.558 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:57.559 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:57 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:57.696 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:57 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:58.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:58.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:58.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:58 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.836 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.837 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:46:58.838 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:58.839 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-08-31T02:46:58.840 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:46:58.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:46:58.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:46:58.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:58.841 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:58 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:46:59.047+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:59.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:59.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:46:59.202 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:46:59.203 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:46:59.204 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:46:59.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:46:59.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:46:59.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:46:59.205 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:46:59.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.579 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.580 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.581 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:46:59.582 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.583 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:59.584 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:46:59 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:46:59.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:46:59.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:46:59.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:59.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:46:59.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:46:59.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:46:59.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:46:59.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:46:59 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.569 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.570 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:47:00.571 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.572 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all crash 2024-08-31T02:47:00.573 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mds 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all nfs 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all grafana 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all loki 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all promtail 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Finalizing container_image settings 2024-08-31T02:47:00.574 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Upgrade: Complete! 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:00.575 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:00 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.699 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:00.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.702 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.703 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:00.704 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:00.705 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.706 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.707 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all crash 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all mds 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:47:00.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:00.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:00.710 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all crash 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all mds 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:47:00.711 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all nfs 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all grafana 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all loki 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all promtail 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Finalizing container_image settings 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Upgrade: Complete! 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.712 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:47:00.713 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all nfs 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all grafana 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all loki 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all promtail 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Finalizing container_image settings 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Upgrade: Complete! 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:00.714 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:00.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:00.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:00 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[12201]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[12201]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[17138]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[17138]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:01 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:01 smithi102 ceph-mon[81198]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:01 smithi102 ceph-mon[81198]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:01 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:01 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:01.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:01 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:02.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-mon[17138]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-mon[17138]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.613 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-mon[12201]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.613 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-mon[12201]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:02 smithi102 ceph-mon[81198]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:02 smithi102 ceph-mon[81198]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:02.717+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:02.950 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:02.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:02.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:02.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:02.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:02.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:02.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:02.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:02.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:03.474 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:03.475 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:03.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:03.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:03.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:03.476 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:03 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:03 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:47:04.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:04.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:04.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:04.426+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:04.710 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.711 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:04.712 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.713 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.714 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.715 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:04.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:04.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:04.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:04.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:04 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:05.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:47:05] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:05 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.700 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.701 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:47:05.702 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.703 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.704 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:05 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.705 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:05.706 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.707 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.708 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.709 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:05 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:05.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:05.921 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:05.922 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:05.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:05.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:05.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:05.923 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:06.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[12201]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[17138]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:06 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:06 smithi102 ceph-mon[81198]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:06 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:06 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:06 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:07 smithi070 ceph-mon[12201]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:07.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:07 smithi070 ceph-mon[12201]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:07 smithi070 ceph-mon[17138]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:07.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:07 smithi070 ceph-mon[17138]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:07 smithi102 ceph-mon[81198]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:07 smithi102 ceph-mon[81198]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:09.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:09 smithi070 ceph-mon[12201]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:09.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:09 smithi070 ceph-mon[17138]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:09.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:09 smithi102 ceph-mon[81198]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-08-31T02:47:11.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:11 smithi070 ceph-mon[17138]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-08-31T02:47:11.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:11 smithi070 ceph-mon[12201]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-08-31T02:47:11.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:11 smithi102 ceph-mon[81198]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 748 B/s rd, 0 op/s 2024-08-31T02:47:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:12 smithi102 ceph-mon[81198]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:12 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:12.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:12 smithi070 ceph-mon[17138]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:12.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:12 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:12.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:12 smithi070 ceph-mon[12201]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:12.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:12 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:14 smithi102 ceph-mon[81198]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:14.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:14 smithi070 ceph-mon[12201]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:14.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:14 smithi070 ceph-mon[17138]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:15.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:15 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:15.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:15 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:15.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:15 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:47:15] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-08-31T02:47:15.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:15 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:17.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:17 smithi070 ceph-mon[12201]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:17.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:17 smithi070 ceph-mon[17138]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:17.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:17 smithi102 ceph-mon[81198]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:19.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:19 smithi070 ceph-mon[12201]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:19.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:19 smithi070 ceph-mon[17138]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:19.724 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:19 smithi102 ceph-mon[81198]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:20.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:47:20.571 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:20 smithi070 ceph-mon[12201]: from='client.25756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:20.571 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:20 smithi070 ceph-mon[17138]: from='client.25756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:20.708 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:20 smithi102 ceph-mon[81198]: from='client.25756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[17138]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[12201]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:21 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:21 smithi102 ceph-mon[81198]: pgmap v389: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:47:21.738 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:21 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.739 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:21 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:21.739 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:21 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:22 smithi070 ceph-mon[12201]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:22.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:22 smithi070 ceph-mon[12201]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:22 smithi070 ceph-mon[17138]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:22.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:22 smithi070 ceph-mon[17138]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:22.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:22 smithi102 ceph-mon[81198]: pgmap v390: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:47:22.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:22 smithi102 ceph-mon[81198]: pgmap v391: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:23.105 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 17s ago 33m - - 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 2s ago 32m - - 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (2m) 17s ago 27m 27.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (2m) 2s ago 27m 81.3M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 17s ago 26m 58.1M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (5m) 2s ago 32m 445M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (16m) 17s ago 33m 528M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (33m) 17s ago 33m 105M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (42s) 2s ago 32m 31.3M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (32m) 17s ago 32m 75.2M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (4m) 17s ago 28m 15.9M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (4m) 2s ago 28m 15.3M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:47:23.106 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (31m) 17s ago 31m 57.6M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (31m) 17s ago 31m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (31m) 17s ago 31m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (30m) 17s ago 30m 53.5M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (30m) 2s ago 30m 56.9M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (29m) 2s ago 29m 52.0M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (29m) 2s ago 29m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (29m) 2s ago 29m 58.1M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (3m) 2s ago 28m 66.8M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (26m) 17s ago 26m 79.5M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:47:23.107 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (26m) 2s ago 26m 78.7M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:47:23.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:23 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:23.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:23 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:23.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:23 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:24.055 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-08-31T02:47:24.608 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:24 smithi070 ceph-mon[17138]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:24.608 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:24 smithi070 ceph-mon[17138]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:24.608 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:24 smithi070 ceph-mon[12201]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:24.609 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:24 smithi070 ceph-mon[12201]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:24.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:24 smithi102 ceph-mon[81198]: pgmap v392: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:47:24.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:24 smithi102 ceph-mon[81198]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:25.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:47:25] "GET /metrics HTTP/1.1" 200 37994 "" "Prometheus/2.43.0" 2024-08-31T02:47:25.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 719 B/s rd, 0 op/s 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 719 B/s rd, 0 op/s 2024-08-31T02:47:25.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-08-31T02:47:25.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:25 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: pgmap v393: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 0 op/s 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: pgmap v394: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 719 B/s rd, 0 op/s 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: pgmap v395: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1000 B/s rd, 0 op/s 2024-08-31T02:47:26.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:25 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:26.215 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:47:26.941 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:26 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3926541578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:26.941 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:26 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3926541578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:26.965 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade status' 2024-08-31T02:47:27.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:26 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/3926541578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:27.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:27 smithi070 ceph-mon[12201]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-08-31T02:47:27.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:27 smithi070 ceph-mon[17138]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-08-31T02:47:28.069 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:27 smithi102 ceph-mon[81198]: pgmap v396: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 850 B/s rd, 0 op/s 2024-08-31T02:47:29.080 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:47:29.080 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": null, 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": false, 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "", 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": null, 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:47:29.081 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:47:29.630 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:47:30.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:29 smithi070 ceph-mon[12201]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-08-31T02:47:30.062 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:29 smithi070 ceph-mon[12201]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:30.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:29 smithi070 ceph-mon[17138]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-08-31T02:47:30.062 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:29 smithi070 ceph-mon[17138]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:30.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:29 smithi102 ceph-mon[81198]: pgmap v397: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-08-31T02:47:30.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:29 smithi102 ceph-mon[81198]: from='client.15864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:30.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:30 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:30.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:30 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:30 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:31.753 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:47:31.753 INFO:teuthology.orchestra.run.smithi070.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:47:31.753 INFO:teuthology.orchestra.run.smithi070.stdout: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:47:32.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-08-31T02:47:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:32 smithi102 ceph-mon[81198]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:32 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/3034510833' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:47:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:32 smithi070 ceph-mon[12201]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:32.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:32 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/3034510833' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:47:32.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:32 smithi070 ceph-mon[17138]: pgmap v398: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:32.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:32 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/3034510833' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:47:33.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:33 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:33.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:33 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:33.753 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:33 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:34.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:34 smithi070 ceph-mon[12201]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:34.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:34 smithi070 ceph-mon[17138]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:34.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:34 smithi102 ceph-mon[81198]: pgmap v399: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:35.886 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:35 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:47:35] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-08-31T02:47:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[12201]: from='client.25786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[12201]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:36.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[12201]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi070", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[17138]: from='client.25786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[17138]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:36.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:36 smithi070 ceph-mon[17138]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi070", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:36 smithi102 ceph-mon[81198]: from='client.25786 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:36 smithi102 ceph-mon[81198]: pgmap v400: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:47:36.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:36 smithi102 ceph-mon[81198]: from='client.15882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "mon", "hosts": "smithi070", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:36.980 INFO:teuthology.orchestra.run.smithi070.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:37.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[12201]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:38.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:37 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: pgmap v401: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: pgmap v402: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 997 B/s rd, 0 op/s 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: pgmap v403: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:38.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:37 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:40.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:40 smithi102 ceph-mon[81198]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-08-31T02:47:40.388 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-mon[12201]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-08-31T02:47:40.388 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-mon[17138]: pgmap v404: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 619 B/s rd, 0 op/s 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:40.437+0000 7f8ca0c9a640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:40.672 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:40.673 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:40.674 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:40.675 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:40.676 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:40.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:40.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:40.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:40.677 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.014 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:47:41.680 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:47:41.680 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 16s ago 33m - - 2024-08-31T02:47:41.680 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 0s ago 33m - - 2024-08-31T02:47:41.680 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (3m) 16s ago 28m 27.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:47:41.680 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (2m) 0s ago 27m 81.4M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 16s ago 26m 58.1M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (5m) 0s ago 32m 445M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (16m) 16s ago 34m 528M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (34m) 16s ago 34m 106M 2048M 17.2.0 e1d6a67b021e 6d3ca0a3deb8 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (60s) 0s ago 32m 32.0M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (32m) 16s ago 32m 75.5M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (4m) 16s ago 28m 15.8M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (4m) 0s ago 28m 15.4M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (32m) 16s ago 32m 57.6M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (31m) 16s ago 31m 58.9M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (31m) 16s ago 31m 50.6M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (31m) 16s ago 30m 53.5M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:47:41.681 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (30m) 0s ago 30m 56.9M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (30m) 0s ago 30m 52.0M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (29m) 0s ago 29m 54.4M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (29m) 0s ago 29m 58.1M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (3m) 0s ago 28m 66.8M - 2.43.0 a07b618ecd1d 7430d008e3cb 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (27m) 16s ago 27m 79.5M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:47:41.682 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (26m) 0s ago 26m 78.8M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:47:41.682 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[12201]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.682 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.683 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:41.684 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.685 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.686 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:41.687 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:41.688 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:41.688 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:41.688 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:41.688 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:41.688 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.689 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.690 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:41.691 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:41.692 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.693 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:41 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:41 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.729 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:41.730 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:47:41.731 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:41.732 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:41.733 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:41 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.734 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:41 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:41.734 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:41 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 1 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:47:42.082 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "mds": {}, 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 3 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:47:42.083 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:47:42.440 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:47:42.440 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi070", 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "", 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b image", 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:47:42.441 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:47:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:42.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[12201]: from='client.? 172.21.15.70:0/213277394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:42 smithi070 ceph-mon[17138]: from='client.? 172.21.15.70:0/213277394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: pgmap v405: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: pgmap v406: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: from='client.15888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: from='client.25804 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:42 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/213277394' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:43.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[12201]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:43.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[12201]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:43.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[12201]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[17138]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:43.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:43 smithi070 ceph-mon[17138]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:43.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:43 smithi102 ceph-mon[81198]: from='client.15900 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:43.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:43 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:43.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:43 smithi102 ceph-mon[81198]: from='client.15912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:47:44.562 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 838 B/s rd, 0 op/s 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 838 B/s rd, 0 op/s 2024-08-31T02:47:44.563 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.564 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:44 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: pgmap v407: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 838 B/s rd, 0 op/s 2024-08-31T02:47:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:47:44.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:47:44.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-31T02:47:44.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:44.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:44 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:45.680 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:47:45] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: Upgrade: Setting container_image for all mgr 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: Upgrade: It appears safe to stop mon.a 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: pgmap v409: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: Upgrade: Setting container_image for all mgr 2024-08-31T02:47:45.681 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: Upgrade: It appears safe to stop mon.a 2024-08-31T02:47:45.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: pgmap v409: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.682 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:45 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mgr 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: Upgrade: It appears safe to stop mon.a 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: pgmap v408: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: pgmap v409: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 745 B/s rd, 0 op/s 2024-08-31T02:47:45.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:45 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: Upgrade: Updating mon.a 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: Deploying daemon mon.a on smithi070 2024-08-31T02:47:47.147 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[12201]: pgmap v410: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: Upgrade: Updating mon.a 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: Deploying daemon mon.a on smithi070 2024-08-31T02:47:47.148 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:46 smithi070 ceph-mon[17138]: pgmap v410: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: Upgrade: Updating mon.a 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: Deploying daemon mon.a on smithi070 2024-08-31T02:47:47.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:46 smithi102 ceph-mon[81198]: pgmap v410: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-08-31T02:47:47.813 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:47] ENGINE Bus STOPPING 2024-08-31T02:47:47.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:47 smithi070 systemd[1]: Stopping Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:47:47.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[12197]: 2024-08-31T02:47:47.700+0000 7f4416239700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:47:47.814 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:47 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[12197]: 2024-08-31T02:47:47.700+0000 7f4416239700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-31T02:47:48.075 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:47 smithi070 podman[104420]: 2024-08-31 02:47:47.81316249 +0000 UTC m=+0.208338308 container died 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, version=8) 2024-08-31T02:47:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 podman[104420]: 2024-08-31 02:47:48.275345355 +0000 UTC m=+0.670521215 container cleanup 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-08-31T02:47:48.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 bash[104420]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a 2024-08-31T02:47:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:47:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:48] ENGINE Bus STOPPED 2024-08-31T02:47:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:48] ENGINE Bus STARTING 2024-08-31T02:47:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:48] ENGINE Serving on http://:::9283 2024-08-31T02:47:48.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:48 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:48] ENGINE Bus STARTED 2024-08-31T02:47:48.800 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 podman[104432]: 2024-08-31 02:47:48.54377578 +0000 UTC m=+0.730985544 container remove 6d3ca0a3deb82a6d0c1191d59e96ef6ab25dfce5d08a10c6745ccee422a325c1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870) 2024-08-31T02:47:48.800 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-31T02:47:48.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 systemd[1]: Stopped Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:47:48.801 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service: Consumed 31.298s CPU time. 2024-08-31T02:47:49.107 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:48 smithi070 systemd[1]: Starting Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:47:49.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 podman[104572]: 2024-08-31 02:47:49.009700841 +0000 UTC m=+0.015970815 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:47:49.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 podman[104572]: 2024-08-31 02:47:49.111491017 +0000 UTC m=+0.117760987 container create 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 podman[104572]: 2024-08-31 02:47:49.364553933 +0000 UTC m=+0.370823910 container init 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 podman[104572]: 2024-08-31 02:47:49.368053953 +0000 UTC m=+0.374323922 container start 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, GIT_BRANCH=HEAD) 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: set uid:gid to 167:167 (ceph:ceph) 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc), process ceph-mon, pid 2 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: pidfile_write: ignore empty --pid-file 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: load: jerasure load: lrc 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: RocksDB version: 7.9.2 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Git sha 0 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Compile date 2024-08-29 18:05:07 2024-08-31T02:47:49.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: DB SUMMARY 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: DB Session ID: QQ8KGBBS2BTTEU0ZCOS2 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: CURRENT file: CURRENT 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: IDENTITY file: IDENTITY 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: MANIFEST file: MANIFEST-000015 size: 4486 Bytes 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000084.sst 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000082.log size: 2358050 ; 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.error_if_exists: 0 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.create_if_missing: 0 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.paranoid_checks: 1 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.env: 0x5560cb898940 2024-08-31T02:47:49.701 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.fs: PosixFileSystem 2024-08-31T02:47:49.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.info_log: 0x5560cdd69560 2024-08-31T02:47:49.702 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_file_opening_threads: 16 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.statistics: (nil) 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.use_fsync: 0 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_log_file_size: 0 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.keep_log_file_num: 1000 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.recycle_log_file_num: 0 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_fallocate: 1 2024-08-31T02:47:49.703 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_mmap_reads: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_mmap_writes: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.use_direct_reads: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.create_missing_column_families: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.db_log_dir: 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.wal_dir: 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-31T02:47:49.704 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.advise_random_on_open: 1 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.db_write_buffer_size: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.write_buffer_manager: 0x5560cdd788c0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.rate_limiter: (nil) 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.wal_recovery_mode: 2 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enable_thread_tracking: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enable_pipelined_write: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.unordered_write: 0 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-31T02:47:49.705 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.row_cache: None 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.wal_filter: None 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_ingest_behind: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.two_write_queues: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.manual_wal_flush: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.wal_compression: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.atomic_flush: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-31T02:47:49.706 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.log_readahead_size: 0 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.best_efforts_recovery: 0 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.allow_data_in_errors: 0 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.db_host_id: __hostname__ 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_background_jobs: 2 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_background_compactions: -1 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_subcompactions: 1 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-31T02:47:49.707 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_total_wal_size: 0 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_open_files: -1 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bytes_per_sync: 0 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_readahead_size: 0 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_background_flushes: -1 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Compression algorithms supported: 2024-08-31T02:47:49.708 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kZSTD supported: 0 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kXpressCompression supported: 0 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kBZip2Compression supported: 0 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kLZ4Compression supported: 1 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kZlibCompression supported: 1 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kLZ4HCCompression supported: 1 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: kSnappyCompression supported: 1 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-31T02:47:49.709 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.merge_operator: 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_filter: None 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_filter_factory: None 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.sst_partitioner_factory: None 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5560cdd69180) 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_type: 0 2024-08-31T02:47:49.710 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_index_type: 0 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_shortening: 1 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: checksum: 4 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: no_block_cache: 0 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache: 0x5560cdd5d090 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_options: 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: capacity : 536870912 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_shard_bits : 4 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: strict_capacity_limit : 0 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_cache_compressed: (nil) 2024-08-31T02:47:49.711 INFO:journalctl@ceph.mon.a.smithi070.stdout: persistent_cache: (nil) 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size: 4096 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_size_deviation: 10 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_restart_interval: 16 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: index_block_restart_interval: 1 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: metadata_block_size: 4096 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: partition_filters: 0 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: use_delta_encoding: 1 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: filter_policy: bloomfilter 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: whole_key_filtering: 1 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: verify_compression: 0 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: format_version: 5 2024-08-31T02:47:49.712 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_index_compression: 1 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout: block_align: 0 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout: max_auto_readahead_size: 262144 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout: prepopulate_block_cache: 0 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout: initial_auto_readahead_size: 8192 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.write_buffer_size: 33554432 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_write_buffer_number: 2 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression: NoCompression 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression: Disabled 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.prefix_extractor: nullptr 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.num_levels: 7 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-31T02:47:49.713 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.level: 32767 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.strategy: 0 2024-08-31T02:47:49.714 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.enabled: false 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.target_file_size_base: 67108864 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-31T02:47:49.715 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.arena_block_size: 1048576 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-31T02:47:49.716 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.disable_auto_compactions: 0 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.inplace_update_support: 0 2024-08-31T02:47:49.717 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.bloom_locality: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.max_successive_merges: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.paranoid_file_checks: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.force_consistency_checks: 1 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.report_bg_io_stats: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.ttl: 2592000 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-31T02:47:49.718 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enable_blob_files: false 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.min_blob_size: 0 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_file_size: 268435456 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.blob_file_starting_level: 0 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 84.sst 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 86, last_sequence is 40120, log_number is 82,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-31T02:47:49.719 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 82 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3f9bc000-d3bb-42c9-aad3-42ec4ce98def 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072469394439, "job": 1, "event": "recovery_started", "wal_files": [82]} 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #82 mode 2 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072469401880, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 87, "file_size": 1880628, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 40121, "largest_seqno": 42100, "table_properties": {"data_size": 1872363, "index_size": 4662, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2565, "raw_key_size": 24131, "raw_average_key_size": 23, "raw_value_size": 1852809, "raw_average_value_size": 1812, "num_data_blocks": 205, "num_entries": 1022, "num_filter_entries": 1022, "num_deletions": 258, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725072469, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3f9bc000-d3bb-42c9-aad3-42ec4ce98def", "db_session_id": "QQ8KGBBS2BTTEU0ZCOS2", "orig_file_number": 87, "seqno_to_time_mapping": "N/A"}} 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072469401993, "job": 1, "event": "recovery_finished"} 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:5047] Creating manifest 89 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000082.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5560cdd8ae00 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: DB pointer 0x5560cde86000 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-31T02:47:49.720 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** DB Stats ** 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-31T02:47:49.721 INFO:journalctl@ceph.mon.a.smithi070.stdout: L0 1/0 1.79 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: L6 1/0 11.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Sum 2/0 13.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 487.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 487.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-08-31T02:47:49.722 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: Cumulative compaction: 0.00 GB write, 180.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: Interval compaction: 0.00 GB write, 180.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: Block cache BinnedLRUCache@0x5560cdd5d090#2 capacity: 512.00 MB usage: 7.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: starting mon.a rank 0 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???) e3 preinit fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).mds e1 new map 2024-08-31T02:47:49.723 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).mds e1 print_map 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: e1 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: legacy client fscid: -1 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout: No filesystems configured 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 ceph-mon[104591]: mon.a@-1(???).paxosservice(auth 1..32) refresh upgraded, format 0 -> 3 2024-08-31T02:47:49.724 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 bash[104572]: 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 2024-08-31T02:47:49.725 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:49 smithi070 systemd[1]: Started Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:47:51.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ignoring --setuser ceph since I am not root 2024-08-31T02:47:51.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ignoring --setgroup ceph since I am not root 2024-08-31T02:47:51.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mgr[64799]: -- 172.21.15.102:0/3770591853 <== mon.2 v2:172.21.15.102:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a11611b0e0 con 0x55a1160f8800 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:50.997+0000 7ffb73bbe040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.053+0000 7ffb73bbe040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: pgmap v411: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: pgmap v412: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: mon.a calling monitor election 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:47:51.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: monmap epoch 3 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: last_changed 2024-08-31T02:14:54.092049+0000 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: min_mon_release 17 (quincy) 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: election_strategy: 1 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: fsmap 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: mgrmap e45: y(active, since 6m), standbys: x 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:47:51.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:47:51.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: from='mgr.15279 ' entity='' 2024-08-31T02:47:51.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:50 smithi102 ceph-mon[81198]: mgrmap e46: y(active, since 6m), standbys: x 2024-08-31T02:47:51.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setuser ceph since I am not root 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setgroup ceph since I am not root 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mgr[56919]: -- 172.21.15.70:0/951197217 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563f5c9450e0 con 0x563f5c923400 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.000+0000 7efd2f981040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.059+0000 7efd2f981040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: pgmap v411: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:47:51.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: pgmap v412: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: mon.a calling monitor election 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: monmap epoch 3 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: last_changed 2024-08-31T02:14:54.092049+0000 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: min_mon_release 17 (quincy) 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: election_strategy: 1 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: fsmap 2024-08-31T02:47:51.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: mgrmap e45: y(active, since 6m), standbys: x 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: from='mgr.15279 ' entity='' 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[17138]: mgrmap e46: y(active, since 6m), standbys: x 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: from='mgr.15279 172.21.15.70:0/1891557876' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: pgmap v411: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:47:51.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: pgmap v412: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 567 B/s rd, 0 op/s 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: mon.a calling monitor election 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: monmap epoch 3 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: last_changed 2024-08-31T02:14:54.092049+0000 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: min_mon_release 17 (quincy) 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: election_strategy: 1 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: fsmap 2024-08-31T02:47:51.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: osdmap e113: 8 total, 8 up, 8 in 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: mgrmap e45: y(active, since 6m), standbys: x 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: from='mgr.15279 ' entity='' 2024-08-31T02:47:51.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:50 smithi070 ceph-mon[104591]: mgrmap e46: y(active, since 6m), standbys: x 2024-08-31T02:47:51.551 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.199+0000 7ffb73bbe040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:47:51.551 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.264+0000 7ffb73bbe040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:47:51.551 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.410+0000 7ffb73bbe040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:47:51.551 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.550+0000 7ffb73bbe040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:47:51.563 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.208+0000 7efd2f981040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:47:51.563 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.274+0000 7efd2f981040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:47:51.563 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.421+0000 7efd2f981040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:47:51.563 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.562+0000 7efd2f981040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:47:52.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.864+0000 7ffb73bbe040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:47:52.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:51.925+0000 7ffb73bbe040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:47:52.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:52.107+0000 7ffb73bbe040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:47:52.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.879+0000 7efd2f981040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:47:52.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:51 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:51.940+0000 7efd2f981040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:47:52.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:52 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:52.124+0000 7efd2f981040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:47:53.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:52 smithi102 ceph-mon[81198]: mgrmap e47: y(active, since 6m), standbys: x 2024-08-31T02:47:53.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:52.994+0000 7ffb73bbe040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:47:53.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:53.060+0000 7ffb73bbe040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:47:53.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:52 smithi070 ceph-mon[104591]: mgrmap e47: y(active, since 6m), standbys: x 2024-08-31T02:47:53.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:53.012+0000 7efd2f981040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:47:53.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:53.079+0000 7efd2f981040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:47:53.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:52 smithi070 ceph-mon[17138]: mgrmap e47: y(active, since 6m), standbys: x 2024-08-31T02:47:53.789 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:47:53.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:47:53.789 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:47:53.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:47:53.966 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:53.776+0000 7ffb73bbe040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:47:53.966 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:53.893+0000 7ffb73bbe040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:47:53.966 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:47:53.683Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-08-31T02:47:53.967 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:47:53.684Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-08-31T02:47:53.967 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:47:53.684Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-08-31T02:47:53.967 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:47:53.685Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-08-31T02:47:53.967 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:47:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:47:53.685Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.70:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.70:8765: connect: connection refused" 2024-08-31T02:47:54.049 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:53.788+0000 7efd2f981040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:47:54.049 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:53 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:53.906+0000 7efd2f981040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:47:54.266 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.048+0000 7ffb73bbe040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:47:54.266 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.115+0000 7ffb73bbe040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:47:54.329 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.048+0000 7efd2f981040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:47:54.329 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.110+0000 7efd2f981040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:47:54.329 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.259+0000 7efd2f981040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:47:54.330 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.328+0000 7efd2f981040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:47:54.556 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.265+0000 7ffb73bbe040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:47:54.557 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.334+0000 7ffb73bbe040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:47:54.557 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.393+0000 7ffb73bbe040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:47:54.557 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.555+0000 7ffb73bbe040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:47:54.619 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.388+0000 7efd2f981040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:47:54.619 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.553+0000 7efd2f981040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:47:54.619 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.618+0000 7efd2f981040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:47:54.832 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.621+0000 7ffb73bbe040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:47:54.832 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.688+0000 7ffb73bbe040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:47:54.832 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.770+0000 7ffb73bbe040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:47:54.832 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.831+0000 7ffb73bbe040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:47:54.885 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.685+0000 7efd2f981040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:47:54.886 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.766+0000 7efd2f981040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:47:54.886 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.825+0000 7efd2f981040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:47:54.886 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:54.884+0000 7efd2f981040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:47:55.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:54.891+0000 7ffb73bbe040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:55.675+0000 7efd2f981040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:55] ENGINE Bus STARTING 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: CherryPy Checker: 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: The Application mounted at '' has an empty config. 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:55] ENGINE Serving on http://:::9283 2024-08-31T02:47:55.942 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:47:55] ENGINE Bus STARTED 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: Active manager daemon y restarted 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: Activating manager daemon y 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: osdmap e114: 8 total, 8 up, 8 in 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: mgrmap e48: y(active, starting, since 0.00742779s), standbys: x 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: Standby manager daemon x restarted 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: Standby manager daemon x started 2024-08-31T02:47:55.943 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:47:55.944 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[104591]: Manager daemon y is now available 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: Active manager daemon y restarted 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: Activating manager daemon y 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: osdmap e114: 8 total, 8 up, 8 in 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: mgrmap e48: y(active, starting, since 0.00742779s), standbys: x 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:55.945 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: Standby manager daemon x restarted 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: Standby manager daemon x started 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:47:55.946 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:47:55.947 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:47:55.947 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:47:55.947 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:47:55.947 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:47:55.947 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:55 smithi070 ceph-mon[17138]: Manager daemon y is now available 2024-08-31T02:47:55.998 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:55.686+0000 7ffb73bbe040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:47:55.998 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:55] ENGINE Bus STARTING 2024-08-31T02:47:55.998 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: CherryPy Checker: 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: The Application mounted at '' has an empty config. 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:55] ENGINE Serving on http://:::9283 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:55] ENGINE Bus STARTED 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: Active manager daemon y restarted 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: Activating manager daemon y 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: osdmap e114: 8 total, 8 up, 8 in 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: mgrmap e48: y(active, starting, since 0.00742779s), standbys: x 2024-08-31T02:47:55.999 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: Standby manager daemon x restarted 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: Standby manager daemon x started 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:47:56.000 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:47:56.001 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:55 smithi102 ceph-mon[81198]: Manager daemon y is now available 2024-08-31T02:47:56.692 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:47:56.390Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-08-31T02:47:56.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:56.691+0000 7efcce68b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-31T02:47:56.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:47:56.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[104591]: mgrmap e49: y(active, since 1.01061s), standbys: x 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:56.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:56.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:56 smithi070 ceph-mon[17138]: mgrmap e49: y(active, since 1.01061s), standbys: x 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:57.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:57.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:56 smithi102 ceph-mon[81198]: mgrmap e49: y(active, since 1.01061s), standbys: x 2024-08-31T02:47:57.929 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:47:57.774Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:57.929 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:57.930 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:57.931 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:58.071 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.072 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' 2024-08-31T02:47:58.073 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:58.073 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi070.ashysy", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-08-31T02:47:58.073 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:57 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:57.938+0000 7efc87f37640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:47:57] ENGINE Error in HTTPServer.serve 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._connections.run(self.expiration_interval) 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._run(expiration_interval) 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:47:58.200 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: s = self.context.wrap_socket( 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.sslsocket_class._create( 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.do_handshake() 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self._sslobj.do_handshake() 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:47:58.201 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:58.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:58.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: [31/Aug/2024:02:47:57] ENGINE Bus STARTING 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[17138]: mgrmap e50: y(active, since 2s), standbys: x 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:58.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: [31/Aug/2024:02:47:57] ENGINE Bus STARTING 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-mon[104591]: mgrmap e50: y(active, since 2s), standbys: x 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:58.878+0000 7efcc05ef640 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:58.951 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:58.952 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.wait_async( 2024-08-31T02:47:58.953 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return future.result(timeout) 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return self.__get_result() 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise self._exception 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: out, err, code = await self._run_cephadm( 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: raise OrchestratorError( 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Creating ceph-iscsi config... 2024-08-31T02:47:58.954 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: Traceback (most recent call last): 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: return _run_code(code, main_globals, None, 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: exec(code, run_globals) 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:58.955 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:58.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:58.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:58.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:58.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:58.956 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:58 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.091 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: Reconfiguring iscsi.foo.smithi070.ashysy (dependencies changed)... 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: Reconfiguring daemon iscsi.foo.smithi070.ashysy on smithi070 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: [31/Aug/2024:02:47:57] ENGINE Bus STARTING 2024-08-31T02:47:59.092 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:58 smithi102 ceph-mon[81198]: mgrmap e50: y(active, since 2s), standbys: x 2024-08-31T02:47:59.361 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE Bus STOPPING 2024-08-31T02:47:59.361 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:47:59.361 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE Bus STOPPED 2024-08-31T02:47:59.361 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE Bus STARTING 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.220+0000 7efd2a920640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setuser ceph since I am not root 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ignoring --setgroup ceph since I am not root 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mgr[56919]: -- 172.21.15.70:0/516073033 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ec5a18d0e0 con 0x55ec5a16b000 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.370+0000 7f3a39278040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.430+0000 7f3a39278040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-31T02:47:59.577 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.575+0000 7f3a39278040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-31T02:47:59.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE Serving on http://:::9283 2024-08-31T02:47:59.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:47:59] ENGINE Bus STARTED 2024-08-31T02:47:59.934 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.641+0000 7f3a39278040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-31T02:47:59.934 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.790+0000 7f3a39278040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:47:59.934 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: [31/Aug/2024:02:47:57] ENGINE Serving on https://172.21.15.70:7151 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: [31/Aug/2024:02:47:57] ENGINE Error in HTTPServer.serve 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:47:59.935 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: [31/Aug/2024:02:47:58] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: [31/Aug/2024:02:47:58] ENGINE Bus STARTED 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:59.936 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:59.937 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:59.938 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise self._exception 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:59.939 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: [31/Aug/2024:02:47:57] ENGINE Serving on https://172.21.15.70:7151 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: [31/Aug/2024:02:47:57] ENGINE Error in HTTPServer.serve 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:47:59.940 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: [31/Aug/2024:02:47:58] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: [31/Aug/2024:02:47:58] ENGINE Bus STARTED 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:59.941 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:59.942 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.wait_async( 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:47:59.943 INFO:journalctl@ceph.mon.c.smithi070.stdout: return future.result(timeout) 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.__get_result() 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise self._exception 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: raise OrchestratorError( 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Creating ceph-iscsi config... 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:47:59.944 INFO:journalctl@ceph.mon.c.smithi070.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.c.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:59.945 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: Activating manager daemon x 2024-08-31T02:47:59.946 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.64104 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: mgrmap e51: x(active, starting, since 0.00528849s) 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.c.smithi070.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:47:59.947 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: exec(code, run_globals) 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:59.948 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: Activating manager daemon x 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.64104 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: mgrmap e51: x(active, starting, since 0.00528849s) 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:47:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: Manager daemon x is now available 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:47:59.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:47:59.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: Manager daemon x is now available 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:47:59.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: [31/Aug/2024:02:47:57] ENGINE Serving on https://172.21.15.70:7151 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: [31/Aug/2024:02:47:57] ENGINE Error in HTTPServer.serve 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:48:00.169 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: [31/Aug/2024:02:47:58] ENGINE Serving on http://172.21.15.70:8765 2024-08-31T02:48:00.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: [31/Aug/2024:02:47:58] ENGINE Bus STARTED 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:48:00.171 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1134, in _check_daemons 2024-08-31T02:48:00.172 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2473, in _daemon_action 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.wait_async( 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 796, in wait_async 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.event_loop.get_result(coro, timeout) 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: return future.result(timeout) 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 446, in result 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.__get_result() 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/concurrent/futures/_base.py", line 391, in __get_result 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise self._exception 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1370, in _create_daemon 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: out, err, code = await self._run_cephadm( 2024-08-31T02:48:00.173 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1713, in _run_cephadm 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: raise OrchestratorError( 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi070.ashysy ... 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: Creating ceph-iscsi config... 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/iscsi-gateway.cfg 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: Write file: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/iscsi.foo.smithi070.ashysy/tcmu-runner-entrypoint.sh 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: Non-zero exit code 1 from systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: systemctl: stderr See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: return _run_code(code, main_globals, None, 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: exec(code, run_globals) 2024-08-31T02:48:00.174 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5579, in 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 5567, in main 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3049, in command_deploy_from 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3084, in _common_deploy 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 3104, in _deploy_daemon_container 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/__main__.py", line 1124, in deploy_daemon 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/tmp/tmp_fnno_uq.cephadm.build/app/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: RuntimeError: Failed command: systemctl restart ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy: Job for ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service failed because the control process exited with error code. 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout: See "systemctl status ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" and "journalctl -xeu ceph-862b0c56-673e-11ef-bcd4-c7b262605968@iscsi.foo.smithi070.ashysy.service" for details. 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 172.21.15.70:0/164266010' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:48:00.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: Activating manager daemon x 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.64104 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: mgrmap e51: x(active, starting, since 0.00528849s) 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-31T02:48:00.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: Manager daemon x is now available 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi070", "caps": []}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi102", "caps": []}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-31T02:48:00.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:47:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-31T02:48:00.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:47:59 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:47:59.933+0000 7f3a39278040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-31T02:48:00.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:00.286+0000 7f3a39278040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-31T02:48:00.553 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:00.358+0000 7f3a39278040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-31T02:48:00.809 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:00.550+0000 7f3a39278040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-31T02:48:00.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: mgrmap e52: x(active, since 1.00794s) 2024-08-31T02:48:00.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.809 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: mgrmap e52: x(active, since 1.00794s) 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:00.810 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:00 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.884 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.884 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: mgrmap e52: x(active, since 1.00794s) 2024-08-31T02:48:00.884 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.884 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi102", "name": "osd_memory_target"}]: dispatch 2024-08-31T02:48:00.884 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:48:00.790+0000 7ffad1c4a640 -1 log_channel(cephadm) log [ERR] : [31/Aug/2024:02:48:00] ENGINE Error in HTTPServer.serve 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: Traceback (most recent call last): 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: self._connections.run(self.expiration_interval) 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: self._run(expiration_interval) 2024-08-31T02:48:00.885 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: s = self.context.wrap_socket( 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: return self.sslsocket_class._create( 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: self.do_handshake() 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: self._sslobj.do_handshake() 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:48:00.886 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: [31/Aug/2024:02:48:00] ENGINE Bus STARTING 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: [31/Aug/2024:02:48:00] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: [31/Aug/2024:02:48:00] ENGINE Error in HTTPServer.serve 2024-08-31T02:48:01.918 INFO:journalctl@ceph.mon.b.smithi102.stdout: Traceback (most recent call last): 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._run(expiration_interval) 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: s = self.context.wrap_socket( 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: return self.sslsocket_class._create( 2024-08-31T02:48:01.919 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self.do_handshake() 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: self._sslobj.do_handshake() 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: [31/Aug/2024:02:48:00] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: [31/Aug/2024:02:48:00] ENGINE Bus STARTED 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: Cluster is now healthy 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:01.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:01.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:01.510+0000 7f3a39278040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-31T02:48:01.949 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:01.584+0000 7f3a39278040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:48:00] ENGINE Bus STARTING 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:48:00] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:48:00] ENGINE Error in HTTPServer.serve 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:48:01.950 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: self.do_handshake() 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:48:00] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: [31/Aug/2024:02:48:00] ENGINE Bus STARTED 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-31T02:48:01.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: Cluster is now healthy 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: [31/Aug/2024:02:48:00] ENGINE Bus STARTING 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: Updating smithi070:/etc/ceph/ceph.conf 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: Updating smithi102:/etc/ceph/ceph.conf 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: [31/Aug/2024:02:48:00] ENGINE Serving on https://172.21.15.102:7150 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: [31/Aug/2024:02:48:00] ENGINE Error in HTTPServer.serve 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: Traceback (most recent call last): 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._connections.run(self.expiration_interval) 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-31T02:48:01.952 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._run(expiration_interval) 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: s = self.context.wrap_socket( 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: return self.sslsocket_class._create( 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: self.do_handshake() 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: self._sslobj.do_handshake() 2024-08-31T02:48:01.953 INFO:journalctl@ceph.mon.a.smithi070.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: [31/Aug/2024:02:48:00] ENGINE Serving on http://172.21.15.102:8765 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: [31/Aug/2024:02:48:00] ENGINE Bus STARTED 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: Cluster is now healthy 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:01.954 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.364+0000 7f3a39278040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-31T02:48:02.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.491+0000 7f3a39278040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-31T02:48:02.706 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.657+0000 7f3a39278040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-31T02:48:02.987 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.733+0000 7f3a39278040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-31T02:48:02.987 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.911+0000 7f3a39278040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-31T02:48:02.987 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:02.984+0000 7f3a39278040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-31T02:48:02.987 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:02.988 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[104591]: mgrmap e53: x(active, since 3s) 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:02.989 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:48:02.990 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.991 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.991 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:02.991 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:02 smithi070 ceph-mon[17138]: mgrmap e53: x(active, since 3s) 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.conf 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi102:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Updating smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/config/ceph.client.admin.keyring 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.016 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.017 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:02 smithi102 ceph-mon[81198]: mgrmap e53: x(active, since 3s) 2024-08-31T02:48:03.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.051+0000 7f3a39278040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-31T02:48:03.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.225+0000 7f3a39278040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-31T02:48:03.305 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.303+0000 7f3a39278040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-31T02:48:03.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.374+0000 7f3a39278040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-31T02:48:03.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.461+0000 7f3a39278040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-31T02:48:03.587 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.524+0000 7f3a39278040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-31T02:48:03.588 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:03.585+0000 7f3a39278040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:48:03.902 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:48:03.903 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.903 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.903 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.903 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.903 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:03 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-31T02:48:03.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: Reconfiguring daemon prometheus.a on smithi102 2024-08-31T02:48:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:03.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:04.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:48:04.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:48:04.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:04.409+0000 7f3a39278040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-31T02:48:04.449 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:48:04] ENGINE Bus STARTING 2024-08-31T02:48:04.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: CherryPy Checker: 2024-08-31T02:48:04.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: The Application mounted at '' has an empty config. 2024-08-31T02:48:04.450 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: 2024-08-31T02:48:04.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:48:04] ENGINE Serving on http://:::9283 2024-08-31T02:48:04.699 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: [31/Aug/2024:02:48:04] ENGINE Bus STARTED 2024-08-31T02:48:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: Standby manager daemon y started 2024-08-31T02:48:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:48:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:48:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:48:04.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:04 smithi102 ceph-mon[81198]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: Standby manager daemon y started 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:48:05.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[104591]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: Standby manager daemon y started 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-31T02:48:05.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-mon[17138]: from='mgr.? 172.21.15.70:0/3860442245' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-31T02:48:05.201 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:48:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:48:04.810Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2024-08-31T02:48:05.803 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:48:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y[56902]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:48:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-31T02:48:06.051 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:05 smithi102 ceph-mon[81198]: mgrmap e54: x(active, since 5s), standbys: y 2024-08-31T02:48:06.052 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:48:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:05 smithi070 ceph-mon[104591]: mgrmap e54: x(active, since 5s), standbys: y 2024-08-31T02:48:06.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:48:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:05 smithi070 ceph-mon[17138]: mgrmap e54: x(active, since 5s), standbys: y 2024-08-31T02:48:06.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:05 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-31T02:48:06.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:06 smithi070 ceph-mon[104591]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:06.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:06 smithi070 ceph-mon[17138]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:07.147 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:06 smithi102 ceph-mon[81198]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:07.418 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:07 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:48:07.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:07 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:07.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:07 smithi070 ceph-mon[17138]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:08.068 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:07 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.056Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-31T02:48:08.068 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:48:08.069 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-31T02:48:08.069 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-31T02:48:08.069 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:48:08.069 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-31T02:48:08.069 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[72271]: ts=2024-08-31T02:48:08.057Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-31T02:48:08.341 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 podman[85893]: 2024-08-31 02:48:08.067410869 +0000 UTC m=+0.119630950 container died 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 podman[85893]: 2024-08-31 02:48:08.507187621 +0000 UTC m=+0.559407702 container cleanup 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:08.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 bash[85893]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:48:08.926 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:08 smithi070 ceph-mon[104591]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-31T02:48:08.927 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:08 smithi070 ceph-mon[17138]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-31T02:48:09.138 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:08 smithi102 ceph-mon[81198]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-31T02:48:09.139 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 podman[85906]: 2024-08-31 02:48:08.791742324 +0000 UTC m=+0.727040324 container remove 7430d008e3cb51d0aa27ddf3d74db93dad8ea0813d03f698e70abe00b85fad99 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:09.139 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-31T02:48:09.139 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 systemd[1]: Stopped Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:48:09.139 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service: Consumed 2.051s CPU time. 2024-08-31T02:48:09.139 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:08 smithi102 systemd[1]: Starting Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:48:09.405 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 podman[85994]: 2024-08-31 02:48:09.041926633 +0000 UTC m=+0.017327443 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 podman[85994]: 2024-08-31 02:48:09.142651147 +0000 UTC m=+0.118051956 container create 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 podman[85994]: 2024-08-31 02:48:09.353259917 +0000 UTC m=+0.328660726 container init 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 podman[85994]: 2024-08-31 02:48:09.356012813 +0000 UTC m=+0.331413625 container start 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.378Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.378Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.378Z caller=main.go:570 level=info host_details="(Linux 5.14.0-503.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024 x86_64 smithi102 (none))" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.379Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.379Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.381Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.381Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.383Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.383Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.384Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-31T02:48:09.406 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.384Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.461µs 2024-08-31T02:48:09.407 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.384Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-31T02:48:09.407 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.384Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=8 2024-08-31T02:48:09.407 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.405Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=8 2024-08-31T02:48:09.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:09] ENGINE Bus STOPPING 2024-08-31T02:48:09.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.413Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=8 2024-08-31T02:48:09.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.416Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=8 2024-08-31T02:48:09.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.421Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=8 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.423Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=8 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.423Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=8 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.426Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=7 maxSegment=8 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.426Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=8 maxSegment=8 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.426Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.729µs wal_replay_duration=41.719224ms wbl_replay_duration=88ns total_replay_duration=41.759595ms 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.429Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.429Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.429Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 bash[85994]: 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.449Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.498608ms db_storage=882ns remote_storage=1.188µs web_handler=357ns query_engine=599ns scrape=255.294µs scrape_sd=107.046µs notify=14.759µs notify_sd=17.54µs rules=18.773165ms tracing=6.552µs 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.449Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-31T02:48:09.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:48:09.449Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-31T02:48:09.670 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:48:09 smithi102 systemd[1]: Started Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:48:10.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:48:10.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:09] ENGINE Bus STOPPED 2024-08-31T02:48:10.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:09] ENGINE Bus STARTING 2024-08-31T02:48:10.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:10] ENGINE Serving on http://:::9283 2024-08-31T02:48:10.418 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:48:10] ENGINE Bus STARTED 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.847 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-31T02:48:10.848 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: Upgrade: It appears safe to stop mon.c 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 25 op/s 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.849 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.850 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: Upgrade: It appears safe to stop mon.c 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 25 op/s 2024-08-31T02:48:10.851 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:10 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi102:9095"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: Upgrade: It appears safe to stop mon.c 2024-08-31T02:48:10.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:10.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 18 op/s 2024-08-31T02:48:10.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 0 B/s wr, 25 op/s 2024-08-31T02:48:10.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:10 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: Upgrade: Updating mon.c 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:11 smithi102 ceph-mon[81198]: Deploying daemon mon.c on smithi070 2024-08-31T02:48:12.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: Upgrade: Updating mon.c 2024-08-31T02:48:12.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:12.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[17138]: Deploying daemon mon.c on smithi070 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: Upgrade: Updating mon.c 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:11 smithi070 ceph-mon[104591]: Deploying daemon mon.c on smithi070 2024-08-31T02:48:12.649 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:12 smithi070 systemd[1]: Stopping Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:48:12.824 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:48:12.918 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:12 smithi070 ceph-mon[104591]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:12.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:12 smithi070 ceph-mon[17138]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:13.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:12 smithi102 ceph-mon[81198]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:13.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c[17133]: 2024-08-31T02:48:12.916+0000 7fb42f0b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:48:13.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:12 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c[17133]: 2024-08-31T02:48:12.916+0000 7fb42f0b8700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-08-31T02:48:13.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:12 smithi070 podman[108158]: 2024-08-31 02:48:12.977640909 +0000 UTC m=+0.177713263 container died 8d9f3f8a0f65aa74999b20b66a8e45e738a1c1cfa647967bb386f1ea1685c712 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 3s ago 33m - - 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 10s ago 33m - - 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (3m) 3s ago 28m 27.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (2m) 10s ago 28m 81.4M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 3s ago 27m 59.0M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (5m) 10s ago 33m 483M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (17m) 3s ago 34m 437M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (24s) 3s ago 34m 41.4M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (92s) 10s ago 33m 38.9M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (33m) 3s ago 33m 81.3M 2048M 17.2.0 e1d6a67b021e 8d9f3f8a0f65 2024-08-31T02:48:13.500 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 3s ago 29m 16.1M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (5m) 10s ago 29m 16.2M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (32m) 3s ago 32m 57.9M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (32m) 3s ago 32m 59.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (31m) 3s ago 31m 51.0M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (31m) 3s ago 31m 53.6M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (31m) 10s ago 31m 57.1M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (30m) 10s ago 30m 52.3M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (30m) 10s ago 30m 55.2M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (29m) 10s ago 29m 57.7M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 starting - - - - 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (27m) 3s ago 27m 79.9M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:48:13.501 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (27m) 10s ago 27m 79.1M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:48:13.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 podman[108158]: 2024-08-31 02:48:13.374800574 +0000 UTC m=+0.574872919 container cleanup 8d9f3f8a0f65aa74999b20b66a8e45e738a1c1cfa647967bb386f1ea1685c712 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , release=754, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:48:13.502 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 bash[108158]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c 2024-08-31T02:48:13.752 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 podman[108196]: 2024-08-31 02:48:13.609947079 +0000 UTC m=+0.634524503 container remove 8d9f3f8a0f65aa74999b20b66a8e45e738a1c1cfa647967bb386f1ea1685c712 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, vendor=Red Hat, Inc., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64) 2024-08-31T02:48:13.926 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:48:13.926 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:48:13.927 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-08-31T02:48:13.928 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 4 2024-08-31T02:48:13.928 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:48:13.928 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:48:14.168 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-31T02:48:14.169 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 systemd[1]: Stopped Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:48:14.169 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.c.service: Consumed 19.683s CPU time. 2024-08-31T02:48:14.169 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:13 smithi070 systemd[1]: Starting Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:48:14.279 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:48:14.279 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:48:14.279 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:48:14.279 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi070", 2024-08-31T02:48:14.279 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:48:14.280 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "1/2 daemons upgraded", 2024-08-31T02:48:14.280 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-08-31T02:48:14.280 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:48:14.280 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:48:14.430 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 podman[108391]: 2024-08-31 02:48:14.071089358 +0000 UTC m=+0.016067275 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:48:14.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 podman[108391]: 2024-08-31 02:48:14.194284614 +0000 UTC m=+0.139262517 container create 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-08-31T02:48:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 podman[108391]: 2024-08-31 02:48:14.430535238 +0000 UTC m=+0.375513152 container init 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-08-31T02:48:14.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 podman[108391]: 2024-08-31 02:48:14.434359412 +0000 UTC m=+0.379337313 container start 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-31T02:48:14.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: set uid:gid to 167:167 (ceph:ceph) 2024-08-31T02:48:14.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc), process ceph-mon, pid 2 2024-08-31T02:48:14.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: pidfile_write: ignore empty --pid-file 2024-08-31T02:48:14.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: load: jerasure load: lrc 2024-08-31T02:48:14.701 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: RocksDB version: 7.9.2 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Git sha 0 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Compile date 2024-08-29 18:05:07 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: DB SUMMARY 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: DB Session ID: 3YHGPSKBO5KVDJV1W2K7 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: CURRENT file: CURRENT 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: IDENTITY file: IDENTITY 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: MANIFEST file: MANIFEST-000009 size: 4796 Bytes 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000084.sst 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000082.log size: 210323 ; 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.error_if_exists: 0 2024-08-31T02:48:14.702 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.create_if_missing: 0 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.paranoid_checks: 1 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.env: 0x55b00a255940 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.fs: PosixFileSystem 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.info_log: 0x55b00c031560 2024-08-31T02:48:14.703 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_file_opening_threads: 16 2024-08-31T02:48:14.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.statistics: (nil) 2024-08-31T02:48:14.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.use_fsync: 0 2024-08-31T02:48:14.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_log_file_size: 0 2024-08-31T02:48:14.704 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.keep_log_file_num: 1000 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.recycle_log_file_num: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_fallocate: 1 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_mmap_reads: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_mmap_writes: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.use_direct_reads: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.create_missing_column_families: 0 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.db_log_dir: 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.wal_dir: 2024-08-31T02:48:14.705 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.advise_random_on_open: 1 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.db_write_buffer_size: 0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.write_buffer_manager: 0x55b00c0408c0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.rate_limiter: (nil) 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-31T02:48:14.706 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.wal_recovery_mode: 2 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enable_thread_tracking: 0 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enable_pipelined_write: 0 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.unordered_write: 0 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.row_cache: None 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.wal_filter: None 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-31T02:48:14.707 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_ingest_behind: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.two_write_queues: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.manual_wal_flush: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.wal_compression: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.atomic_flush: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.log_readahead_size: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.best_efforts_recovery: 0 2024-08-31T02:48:14.708 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.allow_data_in_errors: 0 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.db_host_id: __hostname__ 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_background_jobs: 2 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_background_compactions: -1 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_subcompactions: 1 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_total_wal_size: 0 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-31T02:48:14.709 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_open_files: -1 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bytes_per_sync: 0 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_readahead_size: 0 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_background_flushes: -1 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Compression algorithms supported: 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kZSTD supported: 0 2024-08-31T02:48:14.710 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kXpressCompression supported: 0 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kBZip2Compression supported: 0 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kLZ4Compression supported: 1 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kZlibCompression supported: 1 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kLZ4HCCompression supported: 1 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: kSnappyCompression supported: 1 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.merge_operator: 2024-08-31T02:48:14.711 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_filter: None 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_filter_factory: None 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.sst_partitioner_factory: None 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b00c031180) 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks: 1 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: pin_top_level_index_and_filter: 1 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_type: 0 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_index_type: 0 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_shortening: 1 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-31T02:48:14.712 INFO:journalctl@ceph.mon.c.smithi070.stdout: checksum: 4 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: no_block_cache: 0 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache: 0x55b00c025090 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_name: BinnedLRUCache 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_options: 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: capacity : 536870912 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: num_shard_bits : 4 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: strict_capacity_limit : 0 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: high_pri_pool_ratio: 0.000 2024-08-31T02:48:14.713 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_cache_compressed: (nil) 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: persistent_cache: (nil) 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size: 4096 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_size_deviation: 10 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_restart_interval: 16 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: index_block_restart_interval: 1 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: metadata_block_size: 4096 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: partition_filters: 0 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: use_delta_encoding: 1 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: filter_policy: bloomfilter 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: whole_key_filtering: 1 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: verify_compression: 0 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: read_amp_bytes_per_bit: 0 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: format_version: 5 2024-08-31T02:48:14.714 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_index_compression: 1 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout: block_align: 0 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout: max_auto_readahead_size: 262144 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout: prepopulate_block_cache: 0 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout: initial_auto_readahead_size: 8192 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.write_buffer_size: 33554432 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_write_buffer_number: 2 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression: NoCompression 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression: Disabled 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.prefix_extractor: nullptr 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.num_levels: 7 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-31T02:48:14.715 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.level: 32767 2024-08-31T02:48:14.716 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.strategy: 0 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.enabled: false 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.target_file_size_base: 67108864 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-31T02:48:14.717 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-31T02:48:14.720 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.arena_block_size: 1048576 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.disable_auto_compactions: 0 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-31T02:48:14.721 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.inplace_update_support: 0 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-31T02:48:14.722 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.bloom_locality: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.max_successive_merges: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.paranoid_file_checks: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.force_consistency_checks: 1 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.report_bg_io_stats: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.ttl: 2592000 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-31T02:48:14.723 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enable_blob_files: false 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.min_blob_size: 0 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_file_size: 268435456 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.blob_file_starting_level: 0 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 84.sst 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:48:14.724 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 86, last_sequence is 47072, log_number is 82,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 82 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1ba89666-5ebe-4df5-8749-9fd6f14b3060 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072494461271, "job": 1, "event": "recovery_started", "wal_files": [82]} 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #82 mode 2 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072494462623, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 87, "file_size": 140216, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 47065, "largest_seqno": 47175, "table_properties": {"data_size": 138612, "index_size": 374, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 197, "raw_key_size": 1996, "raw_average_key_size": 28, "raw_value_size": 136907, "raw_average_value_size": 1984, "num_data_blocks": 15, "num_entries": 69, "num_filter_entries": 69, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1725072494, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1ba89666-5ebe-4df5-8749-9fd6f14b3060", "db_session_id": "3YHGPSKBO5KVDJV1W2K7", "orig_file_number": 87, "seqno_to_time_mapping": "N/A"}} 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: EVENT_LOG_v1 {"time_micros": 1725072494462718, "job": 1, "event": "recovery_finished"} 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:5047] Creating manifest 89 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000082.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b00c052e00 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: DB pointer 0x55b00c14e000 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-31T02:48:14.725 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** DB Stats ** 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-31T02:48:14.726 INFO:journalctl@ceph.mon.c.smithi070.stdout: L0 1/0 136.93 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: L6 1/0 11.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Sum 2/0 12.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** Compaction Stats [default] ** 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.5 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.727 INFO:journalctl@ceph.mon.c.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Cumulative compaction: 0.00 GB write, 37.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Interval compaction: 0.00 GB write, 37.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Block cache BinnedLRUCache@0x55b00c025090#2 capacity: 512.00 MB usage: 0.69 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 7e-06 secs_since: 0 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.23 KB,4.47035e-05%) IndexBlock(1,0.45 KB,8.64267e-05%) Misc(1,0.00 KB,0%) 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-31T02:48:14.728 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: starting mon.c rank 1 at public addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] at bind addrs [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???) e3 preinit fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).mds e1 new map 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).mds e1 print_map 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: e1 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: legacy client fscid: -1 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout: No filesystems configured 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).osd e115 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).osd e115 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:48:14.729 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).osd e115 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:48:14.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).osd e115 crush map has features 432629239337189376, adjusting msgr requires 2024-08-31T02:48:14.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 ceph-mon[108413]: mon.c@-1(???).paxosservice(auth 1..33) refresh upgraded, format 0 -> 3 2024-08-31T02:48:14.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 bash[108391]: 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b 2024-08-31T02:48:14.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:14 smithi070 systemd[1]: Started Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:48:19.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:48:19.474+0000 7ffb06630640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-31T02:48:21.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mon.c calling monitor election 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mon.b calling monitor election 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: monmap epoch 4 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: min_mon_release 19 (squid) 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: election_strategy: 1 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: fsmap 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mon.c calling monitor election 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mon.a calling monitor election 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: monmap epoch 4 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: min_mon_release 19 (squid) 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: election_strategy: 1 2024-08-31T02:48:21.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: fsmap 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mon.c calling monitor election 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mon.b calling monitor election 2024-08-31T02:48:21.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: monmap epoch 4 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: min_mon_release 19 (squid) 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: election_strategy: 1 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: fsmap 2024-08-31T02:48:21.455 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mon.c calling monitor election 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mon.a calling monitor election 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: monmap epoch 4 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: min_mon_release 19 (squid) 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: election_strategy: 1 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: fsmap 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:21.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: Reconfiguring mon.a (monmap changed)... 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[104591]: Reconfiguring daemon mon.a on smithi070 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: Reconfiguring mon.a (monmap changed)... 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:21 smithi070 ceph-mon[108413]: Reconfiguring daemon mon.a on smithi070 2024-08-31T02:48:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mon.c calling monitor election 2024-08-31T02:48:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mon.b calling monitor election 2024-08-31T02:48:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: monmap epoch 4 2024-08-31T02:48:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: min_mon_release 19 (squid) 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: election_strategy: 1 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: fsmap 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mon.c calling monitor election 2024-08-31T02:48:21.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mon.a calling monitor election 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: monmap epoch 4 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: fsid 862b0c56-673e-11ef-bcd4-c7b262605968 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: last_changed 2024-08-31T02:48:15.230670+0000 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: created 2024-08-31T02:13:21.834661+0000 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: min_mon_release 19 (squid) 2024-08-31T02:48:21.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: election_strategy: 1 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 0: [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon.a 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 1: [v2:172.21.15.70:3301/0,v1:172.21.15.70:6790/0] mon.c 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: 2: [v2:172.21.15.102:3300/0,v1:172.21.15.102:6789/0] mon.b 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: fsmap 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: osdmap e115: 8 total, 8 up, 8 in 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: mgrmap e54: x(active, since 21s), standbys: y 2024-08-31T02:48:21.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in unknown state 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: Reconfiguring mon.a (monmap changed)... 2024-08-31T02:48:21.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:21.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:21 smithi102 ceph-mon[81198]: Reconfiguring daemon mon.a on smithi070 2024-08-31T02:48:23.069 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: Reconfiguring mgr.y (monmap changed)... 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[104591]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: Reconfiguring mgr.y (monmap changed)... 2024-08-31T02:48:23.070 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:23.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:23.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:23.071 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:22 smithi070 ceph-mon[108413]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: Reconfiguring mgr.y (monmap changed)... 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:23.143 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:22 smithi102 ceph-mon[81198]: Reconfiguring daemon mgr.y on smithi070 2024-08-31T02:48:24.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:24.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:24.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:24.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:23 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:48:23] "GET /metrics HTTP/1.1" 200 38006 "" "Prometheus/2.43.0" 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:24.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:23 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: Reconfiguring mon.c (monmap changed)... 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: Reconfiguring daemon mon.c on smithi070 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:48:25.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:24 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: Reconfiguring mon.c (monmap changed)... 2024-08-31T02:48:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-08-31T02:48:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:25.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: Reconfiguring daemon mon.c on smithi070 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: Reconfiguring mon.c (monmap changed)... 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 689 B/s rd, 0 op/s 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: Reconfiguring daemon mon.c on smithi070 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:25.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:48:25.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:24 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:26.086 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:25 smithi102 ceph-mon[81198]: Reconfiguring osd.0 (monmap changed)... 2024-08-31T02:48:26.086 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:25 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.0 on smithi070 2024-08-31T02:48:26.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:25 smithi070 ceph-mon[104591]: Reconfiguring osd.0 (monmap changed)... 2024-08-31T02:48:26.087 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:25 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.0 on smithi070 2024-08-31T02:48:26.087 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:25 smithi070 ceph-mon[108413]: Reconfiguring osd.0 (monmap changed)... 2024-08-31T02:48:26.087 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:25 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.0 on smithi070 2024-08-31T02:48:26.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:26 smithi102 ceph-mon[81198]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:26.980 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:26 smithi070 ceph-mon[104591]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:26.981 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:26 smithi070 ceph-mon[108413]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: Reconfiguring osd.1 (monmap changed)... 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:48:27.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.1 on smithi070 2024-08-31T02:48:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.976 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: Reconfiguring osd.1 (monmap changed)... 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.1 on smithi070 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.977 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: Reconfiguring osd.1 (monmap changed)... 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.1 on smithi070 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:27.978 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:28.968 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[104591]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 744 B/s rd, 0 op/s 2024-08-31T02:48:28.968 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[104591]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-08-31T02:48:28.968 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[104591]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:48:28.968 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[108413]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 744 B/s rd, 0 op/s 2024-08-31T02:48:28.968 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[108413]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-08-31T02:48:28.969 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:28 smithi070 ceph-mon[108413]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:48:29.035 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:28 smithi102 ceph-mon[81198]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 744 B/s rd, 0 op/s 2024-08-31T02:48:29.035 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:28 smithi102 ceph-mon[81198]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-08-31T02:48:29.035 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:28 smithi102 ceph-mon[81198]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1 KiB/s rd, 1 op/s 2024-08-31T02:48:29.954 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:29.954 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:30.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:30.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:30.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.033 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:48:31.033 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.033 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-08-31T02:48:31.033 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:31.034 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:48:31.035 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 765 B/s rd, 0 op/s 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-31T02:48:31.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:31.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: Reconfiguring osd.2 (monmap changed)... 2024-08-31T02:48:31.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.2 on smithi070 2024-08-31T02:48:31.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: Reconfiguring osd.2 (monmap changed)... 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.2 on smithi070 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:48:31.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:32.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: Reconfiguring osd.2 (monmap changed)... 2024-08-31T02:48:32.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.2 on smithi070 2024-08-31T02:48:32.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:32.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:32.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-31T02:48:32.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:33.052 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[104591]: Reconfiguring osd.3 (monmap changed)... 2024-08-31T02:48:33.052 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.3 on smithi070 2024-08-31T02:48:33.052 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[104591]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:33.053 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[108413]: Reconfiguring osd.3 (monmap changed)... 2024-08-31T02:48:33.053 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.3 on smithi070 2024-08-31T02:48:33.053 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:32 smithi070 ceph-mon[108413]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:33.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:32 smithi102 ceph-mon[81198]: Reconfiguring osd.3 (monmap changed)... 2024-08-31T02:48:33.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:32 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.3 on smithi070 2024-08-31T02:48:33.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:32 smithi102 ceph-mon[81198]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:48:33] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-08-31T02:48:34.888 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[104591]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:48:34.888 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:34.889 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:34.889 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:34.889 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:34.890 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[108413]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:48:34.890 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:34.890 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:34.890 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:34.890 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:35.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:34 smithi102 ceph-mon[81198]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 0 op/s 2024-08-31T02:48:35.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:35.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:35.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi070.usqphe", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:35.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: Reconfiguring rgw.foo.smithi070.usqphe (monmap changed)... 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: Reconfiguring daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:36.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:36.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: Reconfiguring rgw.foo.smithi070.usqphe (monmap changed)... 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: Reconfiguring daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:36.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: Reconfiguring rgw.foo.smithi070.usqphe (monmap changed)... 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: Reconfiguring daemon rgw.foo.smithi070.usqphe on smithi070 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-31T02:48:36.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:36.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[108413]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:48:36.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[108413]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:48:36.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[108413]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:36.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[104591]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:48:36.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[104591]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:48:36.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:36 smithi070 ceph-mon[104591]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:37.118 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:36 smithi102 ceph-mon[81198]: Reconfiguring mon.b (monmap changed)... 2024-08-31T02:48:37.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:36 smithi102 ceph-mon[81198]: Reconfiguring daemon mon.b on smithi102 2024-08-31T02:48:37.119 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:36 smithi102 ceph-mon[81198]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:38.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:38.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:37 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:38.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-31T02:48:38.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:37 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:38.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:38.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:38.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: Reconfiguring mgr.x (monmap changed)... 2024-08-31T02:48:38.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: Reconfiguring daemon mgr.x on smithi102 2024-08-31T02:48:38.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:48:38.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:38 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:38.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:38.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:38.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: Reconfiguring mgr.x (monmap changed)... 2024-08-31T02:48:38.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: Reconfiguring daemon mgr.x on smithi102 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: Reconfiguring mgr.x (monmap changed)... 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: Reconfiguring daemon mgr.x on smithi102 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:38.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-31T02:48:38.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:38 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:40.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:39 smithi102 ceph-mon[81198]: Reconfiguring osd.4 (monmap changed)... 2024-08-31T02:48:40.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:39 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.4 on smithi102 2024-08-31T02:48:40.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:39 smithi070 ceph-mon[104591]: Reconfiguring osd.4 (monmap changed)... 2024-08-31T02:48:40.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:39 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.4 on smithi102 2024-08-31T02:48:40.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:39 smithi070 ceph-mon[108413]: Reconfiguring osd.4 (monmap changed)... 2024-08-31T02:48:40.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:39 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.4 on smithi102 2024-08-31T02:48:40.896 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:40 smithi102 ceph-mon[81198]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:40.896 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:40 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:40.896 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:40 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:40.896 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:40 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:48:40.896 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:40 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:41.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[108413]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:41.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[104591]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-31T02:48:41.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:40 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:42.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:41 smithi102 ceph-mon[81198]: Reconfiguring osd.5 (monmap changed)... 2024-08-31T02:48:42.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:41 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.5 on smithi102 2024-08-31T02:48:42.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:41 smithi070 ceph-mon[104591]: Reconfiguring osd.5 (monmap changed)... 2024-08-31T02:48:42.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:41 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.5 on smithi102 2024-08-31T02:48:42.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:41 smithi070 ceph-mon[108413]: Reconfiguring osd.5 (monmap changed)... 2024-08-31T02:48:42.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:41 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.5 on smithi102 2024-08-31T02:48:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:42 smithi102 ceph-mon[81198]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:48:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[104591]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:48:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[108413]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-31T02:48:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: Reconfiguring osd.6 (monmap changed)... 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.6 on smithi102 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:48:44.010 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:44.011 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:48:43] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-08-31T02:48:44.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: Reconfiguring osd.6 (monmap changed)... 2024-08-31T02:48:44.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.6 on smithi102 2024-08-31T02:48:44.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: Reconfiguring osd.6 (monmap changed)... 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.6 on smithi102 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-31T02:48:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:43 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:44.643 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:48:44.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[104591]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:44.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[104591]: Reconfiguring osd.7 (monmap changed)... 2024-08-31T02:48:44.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[104591]: Reconfiguring daemon osd.7 on smithi102 2024-08-31T02:48:44.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[108413]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[108413]: Reconfiguring osd.7 (monmap changed)... 2024-08-31T02:48:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[108413]: Reconfiguring daemon osd.7 on smithi102 2024-08-31T02:48:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:44 smithi102 ceph-mon[81198]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:44 smithi102 ceph-mon[81198]: Reconfiguring osd.7 (monmap changed)... 2024-08-31T02:48:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:44 smithi102 ceph-mon[81198]: Reconfiguring daemon osd.7 on smithi102 2024-08-31T02:48:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:45.294 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 8s ago 34m - - 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 18s ago 34m - - 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 8s ago 29m 27.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (3m) 18s ago 28m 81.4M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 8s ago 27m 61.7M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (6m) 18s ago 33m 497M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (17m) 8s ago 35m 438M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (56s) 8s ago 35m 45.5M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (2m) 18s ago 33m 40.9M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (31s) 8s ago 34m 26.5M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 7a529ea3924f 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (5m) 8s ago 29m 16.2M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (5m) 18s ago 29m 15.7M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:48:45.295 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (33m) 8s ago 33m 57.9M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (32m) 8s ago 32m 59.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (32m) 8s ago 32m 51.3M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (32m) 8s ago 32m 53.7M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (31m) 18s ago 31m 57.3M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (31m) 18s ago 31m 52.3M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (30m) 18s ago 30m 55.2M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (30m) 18s ago 30m 57.8M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (36s) 18s ago 29m 53.9M - 2.43.0 a07b618ecd1d 8c3920246ae0 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (28m) 8s ago 28m 80.3M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:48:45.296 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (28m) 18s ago 28m 79.3M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 3 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:45.686 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 5 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:48:45.687 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:48:45.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='client.74101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:45.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/2500130601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='client.74101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:45.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:45 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/2500130601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.032 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:48:46.032 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:48:46.032 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi070", 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [ 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "mon" 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "2/2 daemons upgraded", 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading mon daemons", 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:48:46.033 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='client.74101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi102.axbapt", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:46.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:45 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/2500130601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: Reconfiguring rgw.foo.smithi102.axbapt (monmap changed)... 2024-08-31T02:48:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: Reconfiguring daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='client.64209 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.920 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.921 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:48:46.922 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.923 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:46.924 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: Reconfiguring rgw.foo.smithi102.axbapt (monmap changed)... 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: Reconfiguring daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='client.64209 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.091 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.092 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: Reconfiguring rgw.foo.smithi102.axbapt (monmap changed)... 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: Reconfiguring daemon rgw.foo.smithi102.axbapt on smithi102 2024-08-31T02:48:47.093 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='client.64203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 549 B/s rd, 0 op/s 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='client.64209 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.094 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.095 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.096 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:48:47.097 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.098 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:48:47.099 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:48:47.100 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.101 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-31T02:48:47.102 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:48:47.103 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:48:47.104 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: from='client.74125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:48.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mon 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all crash 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mds 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all nfs 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all grafana 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all loki 2024-08-31T02:48:48.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all promtail 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Finalizing container_image settings 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: Upgrade: Complete! 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:47 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: from='client.74125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all mon 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all crash 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all mds 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all nfs 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:48:48.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all grafana 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all loki 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all promtail 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Finalizing container_image settings 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: Upgrade: Complete! 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: from='client.74125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all mon 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all crash 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all mds 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all rbd-mirror 2024-08-31T02:48:48.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all ceph-exporter 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all nfs 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all node-exporter 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all prometheus 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all alertmanager 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all grafana 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all loki 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all promtail 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Finalizing container_image settings 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: Upgrade: Complete! 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:48:48.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-08-31T02:48:48.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:48.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:48.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:47 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:49.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:48 smithi102 ceph-mon[81198]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:49.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:48 smithi102 ceph-mon[81198]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:48:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:48 smithi070 ceph-mon[104591]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:49.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:48 smithi070 ceph-mon[104591]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:48:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:48 smithi070 ceph-mon[108413]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:49.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:48 smithi070 ceph-mon[108413]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:48:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:50 smithi102 ceph-mon[81198]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:50.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:50 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:50 smithi070 ceph-mon[104591]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:50.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:50 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:50.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:50 smithi070 ceph-mon[108413]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:48:50.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:50 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:48:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:52 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:48:52.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:48:52.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:52 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:48:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:48:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:48:53] "GET /metrics HTTP/1.1" 200 38014 "" "Prometheus/2.43.0" 2024-08-31T02:48:54.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:54 smithi070 ceph-mon[104591]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:48:54.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:54 smithi070 ceph-mon[108413]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:48:54.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:54 smithi102 ceph-mon[81198]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:48:56.760 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:56 smithi070 ceph-mon[104591]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:56.761 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:56 smithi070 ceph-mon[108413]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:56.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:56 smithi102 ceph-mon[81198]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:57.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:57 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:57.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:57 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:57.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:57 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:48:58.819 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:58 smithi102 ceph-mon[81198]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:58.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:58 smithi070 ceph-mon[104591]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:58.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:58 smithi070 ceph-mon[108413]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:48:59.794 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:48:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:59.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:48:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:48:59.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:48:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:00.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:00 smithi102 ceph-mon[81198]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:49:00.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:00 smithi070 ceph-mon[104591]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:49:00.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:00 smithi070 ceph-mon[108413]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 636 B/s rd, 0 op/s 2024-08-31T02:49:02.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:02 smithi102 ceph-mon[81198]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:02.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:02 smithi070 ceph-mon[104591]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:02.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:02 smithi070 ceph-mon[108413]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:03] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-08-31T02:49:04.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:04 smithi102 ceph-mon[81198]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:04 smithi070 ceph-mon[104591]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:04.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:04 smithi070 ceph-mon[108413]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:06.771 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:06 smithi070 ceph-mon[104591]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:06.771 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:06 smithi070 ceph-mon[108413]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:06.793 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:06 smithi102 ceph-mon[81198]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:07.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:07 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:07.931 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:07 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:07.932 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:07 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-08-31T02:49:08.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:08.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:08 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1015 B/s rd, 0 op/s 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:08.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:08 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:10.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:10 smithi102 ceph-mon[81198]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-08-31T02:49:10.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:10 smithi070 ceph-mon[104591]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-08-31T02:49:10.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:10 smithi070 ceph-mon[108413]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 633 B/s rd, 0 op/s 2024-08-31T02:49:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:49:12.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:12 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 836 B/s rd, 0 op/s 2024-08-31T02:49:12.952 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:12.953 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:12 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:13] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-08-31T02:49:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:14 smithi102 ceph-mon[81198]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:49:14.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:14 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:14.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:14 smithi070 ceph-mon[104591]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:49:14.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:14 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:14.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:14 smithi070 ceph-mon[108413]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 848 B/s rd, 0 op/s 2024-08-31T02:49:14.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:14 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:16.698 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:16 smithi070 ceph-mon[104591]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:16.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:16 smithi070 ceph-mon[108413]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:16.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:16 smithi102 ceph-mon[81198]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:17.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:49:17.730 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:17 smithi070 ceph-mon[104591]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:17.730 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:17 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:17.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:17 smithi070 ceph-mon[108413]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:17.730 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:17 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:17.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:17 smithi102 ceph-mon[81198]: from='client.74131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:17.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:17 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:18.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:18 smithi102 ceph-mon[81198]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:18.920 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:18 smithi070 ceph-mon[104591]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:18.920 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:18 smithi070 ceph-mon[108413]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 7s ago 34m - - 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 11s ago 34m - - 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (4m) 7s ago 29m 26.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (4m) 11s ago 29m 81.9M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 unknown 7s ago 28m 61.7M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (7m) 11s ago 34m 502M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (18m) 7s ago 35m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (90s) 7s ago 35m 49.3M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (2m) 11s ago 34m 43.9M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:49:19.366 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (65s) 7s ago 34m 36.1M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 7a529ea3924f 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6m) 7s ago 30m 16.4M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (6m) 11s ago 30m 15.4M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (33m) 7s ago 33m 57.9M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (33m) 7s ago 33m 59.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (33m) 7s ago 33m 51.3M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (32m) 7s ago 32m 53.7M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (32m) 11s ago 32m 57.3M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (31m) 11s ago 31m 52.3M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (31m) 11s ago 31m 55.2M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (31m) 11s ago 31m 57.8M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (70s) 11s ago 30m 55.1M - 2.43.0 a07b618ecd1d 8c3920246ae0 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (28m) 7s ago 28m 80.4M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:49:19.367 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (28m) 11s ago 28m 79.4M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:49:19.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-08-31T02:49:20.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:20 smithi102 ceph-mon[81198]: from='client.74137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:20.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:20 smithi102 ceph-mon[81198]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-08-31T02:49:20.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:20 smithi070 ceph-mon[104591]: from='client.74137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:20.923 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:20 smithi070 ceph-mon[104591]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-08-31T02:49:20.924 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:20 smithi070 ceph-mon[108413]: from='client.74137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:20.924 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:20 smithi070 ceph-mon[108413]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-08-31T02:49:22.045 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:49:22.736 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-08-31T02:49:22.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:22 smithi102 ceph-mon[81198]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:22.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:22 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/3693164263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:22.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:22 smithi070 ceph-mon[104591]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:22.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:22 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/3693164263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:22.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:22 smithi070 ceph-mon[108413]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:22.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:22 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/3693164263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:24.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:23] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-08-31T02:49:24.872 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)" 2024-08-31T02:49:24.873 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:24 smithi070 ceph-mon[104591]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:24.873 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:24 smithi070 ceph-mon[108413]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:24.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:24 smithi102 ceph-mon[81198]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:25.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-08-31T02:49:25.868 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:25 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/3978923699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:25.869 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:25 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/3978923699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:25.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:25 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/3978923699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:26.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:26 smithi102 ceph-mon[81198]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:26.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:26 smithi070 ceph-mon[108413]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:26.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:26 smithi070 ceph-mon[104591]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:27.819 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:27 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:27.883 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:27 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:27.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:27 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.799 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.800 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1012 B/s rd, 0 op/s 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:28.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:29.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:29 smithi102 ceph-mon[81198]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 735 B/s rd, 0 op/s 2024-08-31T02:49:29.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:29 smithi102 ceph-mon[81198]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:29.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:29.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[104591]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 735 B/s rd, 0 op/s 2024-08-31T02:49:29.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[104591]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:29.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:29.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[108413]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 735 B/s rd, 0 op/s 2024-08-31T02:49:29.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[108413]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:29.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:30.904 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:30 smithi070 ceph-mon[104591]: from='client.64251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:30.904 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:30 smithi070 ceph-mon[108413]: from='client.64251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:30.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:30 smithi102 ceph-mon[81198]: from='client.64251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:31.253 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:49:31.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:31 smithi102 ceph-mon[81198]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:31.925 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:31 smithi070 ceph-mon[104591]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:31.925 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:31 smithi070 ceph-mon[108413]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:31.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade status' 2024-08-31T02:49:33.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:33 smithi102 ceph-mon[81198]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:33.918 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:33] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-08-31T02:49:33.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:33 smithi070 ceph-mon[104591]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:33.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:33 smithi070 ceph-mon[108413]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:34.134 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:49:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": null, 2024-08-31T02:49:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": false, 2024-08-31T02:49:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "", 2024-08-31T02:49:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:49:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": null, 2024-08-31T02:49:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "", 2024-08-31T02:49:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:49:34.135 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:49:34.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph health detail' 2024-08-31T02:49:35.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:35 smithi102 ceph-mon[81198]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:35.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:35 smithi102 ceph-mon[81198]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:49:35.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:35 smithi070 ceph-mon[104591]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:35.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:35 smithi070 ceph-mon[104591]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:49:35.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:35 smithi070 ceph-mon[108413]: from='client.74161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:35.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:35 smithi070 ceph-mon[108413]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 746 B/s rd, 0 op/s 2024-08-31T02:49:36.843 INFO:teuthology.orchestra.run.smithi070.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-08-31T02:49:36.844 INFO:teuthology.orchestra.run.smithi070.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:49:36.844 INFO:teuthology.orchestra.run.smithi070.stdout: daemon iscsi.foo.smithi070.ashysy on smithi070 is in error state 2024-08-31T02:49:37.545 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[104591]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/2139799225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[108413]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:37.844 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:37 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/2139799225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:49:37.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:37 smithi102 ceph-mon[81198]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:37.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:37 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:37.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:37 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/2139799225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-08-31T02:49:39.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:39 smithi102 ceph-mon[81198]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:39.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:39 smithi070 ceph-mon[104591]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:39.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:39 smithi070 ceph-mon[108413]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:49:40.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:40 smithi102 ceph-mon[81198]: from='client.74173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:40.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:40 smithi070 ceph-mon[104591]: from='client.74173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:40.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:40 smithi070 ceph-mon[108413]: from='client.74173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:41.564 INFO:teuthology.orchestra.run.smithi070.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:41.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:41 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:41.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:41.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:41.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:41 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:42.660 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-31T02:49:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:42 smithi102 ceph-mon[81198]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:42 smithi102 ceph-mon[81198]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:49:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:42 smithi102 ceph-mon[81198]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:42.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:42 smithi102 ceph-mon[81198]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[108413]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[108413]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[108413]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[108413]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[104591]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[104591]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 923 B/s rd, 0 op/s 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[104591]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:42.919 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:42 smithi070 ceph-mon[104591]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b 2024-08-31T02:49:44.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:43] "GET /metrics HTTP/1.1" 200 38011 "" "Prometheus/2.43.0" 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all mgr 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all mon 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all crash 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: Upgrade: osd.0 is safe to restart 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.951 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all mgr 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all mon 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all crash 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:44.952 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:44.953 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: Upgrade: osd.0 is safe to restart 2024-08-31T02:49:44.953 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:44 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: Target is version 19.1.1-240-g34b9d27b (squid) 2024-08-31T02:49:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769'] 2024-08-31T02:49:45.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mgr 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all mon 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all crash 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:49:45.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: Upgrade: osd.0 is safe to restart 2024-08-31T02:49:45.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:44 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:49:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: Upgrade: Updating osd.0 2024-08-31T02:49:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:46.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: Deploying daemon osd.0 on smithi070 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[104591]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: Upgrade: Updating osd.0 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: Deploying daemon osd.0 on smithi070 2024-08-31T02:49:46.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:46 smithi070 ceph-mon[108413]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:46.753 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:49:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: Upgrade: Updating osd.0 2024-08-31T02:49:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:49:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:46.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: Deploying daemon osd.0 on smithi070 2024-08-31T02:49:46.919 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:46 smithi102 ceph-mon[81198]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:47.419 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:49:47.419 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 18s ago 35m - - 2024-08-31T02:49:47.419 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 19s ago 35m - - 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (5m) 18s ago 30m 26.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (4m) 19s ago 29m 80.3M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 error 18s ago 28m 61.7M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (7m) 19s ago 34m 503M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (18m) 18s ago 36m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (118s) 18s ago 36m 50.3M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (3m) 19s ago 34m 44.5M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (93s) 18s ago 35m 38.1M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 7a529ea3924f 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (6m) 18s ago 30m 16.4M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (6m) 19s ago 30m 16.4M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:49:47.420 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (34m) 18s ago 34m 57.9M 4096M 17.2.0 e1d6a67b021e 1b42d351458e 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (33m) 18s ago 33m 59.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (33m) 18s ago 33m 51.3M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (33m) 18s ago 33m 53.7M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (32m) 19s ago 32m 57.3M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (32m) 19s ago 32m 52.4M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (31m) 19s ago 31m 55.2M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (31m) 19s ago 31m 57.8M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (98s) 19s ago 30m 55.3M - 2.43.0 a07b618ecd1d 8c3920246ae0 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (29m) 18s ago 29m 80.5M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:49:47.421 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (29m) 19s ago 29m 79.5M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:49:47.688 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:47 smithi070 ceph-mon[104591]: from='client.64275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:47.688 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:47 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:47.689 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:47 smithi070 ceph-mon[108413]: from='client.64275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:47.689 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:47 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:47.701 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:47 smithi102 ceph-mon[81198]: from='client.64275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:47.701 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:47 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:47.846 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 3 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:49:47.847 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 5 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:49:47.848 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "0/8 daemons upgraded", 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-08-31T02:49:48.189 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:49:48.190 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:49:48.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='client.64281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:48.875 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='client.74191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 704 B/s rd, 0 op/s 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/1384563244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.876 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='client.64281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='client.74191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 704 B/s rd, 0 op/s 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/1384563244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.877 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:48.878 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:48 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.030 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='client.64281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:49.030 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='client.74191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 704 B/s rd, 0 op/s 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 827 B/s rd, 0 op/s 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/1384563244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.031 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:48 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:49.853 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[104591]: from='client.74203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:49.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[104591]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:49:49.854 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[104591]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:49.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[108413]: from='client.74203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:49.854 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[108413]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:49:49.855 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:49 smithi070 ceph-mon[108413]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:49.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:49 smithi102 ceph-mon[81198]: from='client.74203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:49:49.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:49 smithi102 ceph-mon[81198]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 728 B/s rd, 0 op/s 2024-08-31T02:49:49.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:49 smithi102 ceph-mon[81198]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:50.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:50 smithi070 systemd[1]: Stopping Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:49:50.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[20387]: 2024-08-31T02:49:50.423+0000 7fd7331ea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:49:50.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[20387]: 2024-08-31T02:49:50.423+0000 7fd7331ea700 -1 osd.0 115 *** Got signal Terminated *** 2024-08-31T02:49:50.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:50 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[20387]: 2024-08-31T02:49:50.423+0000 7fd7331ea700 -1 osd.0 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:49:50.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:50 smithi102 ceph-mon[81198]: osd.0 marked itself down and dead 2024-08-31T02:49:51.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:50 smithi070 ceph-mon[108413]: osd.0 marked itself down and dead 2024-08-31T02:49:51.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:50 smithi070 ceph-mon[104591]: osd.0 marked itself down and dead 2024-08-31T02:49:51.591 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:51 smithi070 podman[113310]: 2024-08-31 02:49:51.261881784 +0000 UTC m=+0.926606637 container died 1b42d351458ebc8664ec10d80bd6f99737959d7d8c0a3e7b2cd165233fe53a4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, version=8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754) 2024-08-31T02:49:51.883 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[104591]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:51.884 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[104591]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:49:51.884 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[104591]: osdmap e116: 8 total, 7 up, 8 in 2024-08-31T02:49:51.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[108413]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:51.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[108413]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:49:51.884 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:51 smithi070 ceph-mon[108413]: osdmap e116: 8 total, 7 up, 8 in 2024-08-31T02:49:51.884 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:51 smithi070 podman[113310]: 2024-08-31 02:49:51.690019163 +0000 UTC m=+1.354744010 container cleanup 1b42d351458ebc8664ec10d80bd6f99737959d7d8c0a3e7b2cd165233fe53a4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public) 2024-08-31T02:49:51.884 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:51 smithi070 bash[113310]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0 2024-08-31T02:49:52.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:51 smithi102 ceph-mon[81198]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1017 B/s rd, 0 op/s 2024-08-31T02:49:52.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:51 smithi102 ceph-mon[81198]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:49:52.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:51 smithi102 ceph-mon[81198]: osdmap e116: 8 total, 7 up, 8 in 2024-08-31T02:49:52.199 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:51 smithi070 podman[113327]: 2024-08-31 02:49:51.883167831 +0000 UTC m=+0.618950543 container remove 1b42d351458ebc8664ec10d80bd6f99737959d7d8c0a3e7b2cd165233fe53a4f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, architecture=x86_64, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, release=754, ceph=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8) 2024-08-31T02:49:52.627 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.201999059 +0000 UTC m=+0.016024673 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:49:52.627 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.334160531 +0000 UTC m=+0.148186132 container create 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, RELEASE=squid-34b9d27, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-31T02:49:52.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:52 smithi070 ceph-mon[104591]: osdmap e117: 8 total, 7 up, 8 in 2024-08-31T02:49:52.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:52 smithi070 ceph-mon[108413]: osdmap e117: 8 total, 7 up, 8 in 2024-08-31T02:49:52.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.627071435 +0000 UTC m=+0.441097043 container init 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:49:52.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.634603571 +0000 UTC m=+0.448629173 container start 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-31T02:49:52.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.769771989 +0000 UTC m=+0.583797597 container attach 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-31T02:49:52.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:52 smithi070 podman[113415]: 2024-08-31 02:49:52.826705578 +0000 UTC m=+0.640731198 container died 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731) 2024-08-31T02:49:53.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:52 smithi102 ceph-mon[81198]: osdmap e117: 8 total, 7 up, 8 in 2024-08-31T02:49:53.764 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 podman[113439]: 2024-08-31 02:49:53.465066188 +0000 UTC m=+0.632414853 container remove 8205f214304e3ea6449794e8147fa7ef7994b0e2bf16d95954e882b067c65de5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-31T02:49:53.765 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-31T02:49:53.765 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 systemd[1]: Stopped Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:49:53.765 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service: Consumed 15.197s CPU time. 2024-08-31T02:49:53.765 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 systemd[1]: Starting Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:49:54.028 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:53 smithi070 ceph-mon[104591]: pgmap v97: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:54.029 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:53 smithi070 ceph-mon[108413]: pgmap v97: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:54.029 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 podman[113537]: 2024-08-31 02:49:53.807272918 +0000 UTC m=+0.016005521 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:49:54.029 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:53 smithi070 podman[113537]: 2024-08-31 02:49:53.933295872 +0000 UTC m=+0.142028511 container create 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-31T02:49:54.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:53 smithi102 ceph-mon[81198]: pgmap v97: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:49:54.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:49:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:49:53] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-08-31T02:49:54.317 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 podman[113537]: 2024-08-31 02:49:54.185586333 +0000 UTC m=+0.394318930 container init 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-34b9d27, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-08-31T02:49:54.318 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 podman[113537]: 2024-08-31 02:49:54.18903648 +0000 UTC m=+0.397769079 container start 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-31T02:49:54.318 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 podman[113537]: 2024-08-31 02:49:54.260544073 +0000 UTC m=+0.469276682 container attach 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:49:54.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-31T02:49:54.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-31T02:49:55.031 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[113553]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 bash[113537]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-31T02:49:55.032 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:54 smithi070 podman[113537]: 2024-08-31 02:49:54.747172328 +0000 UTC m=+0.955904926 container died 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27) 2024-08-31T02:49:55.662 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 podman[113671]: 2024-08-31 02:49:55.330887115 +0000 UTC m=+0.574000194 container remove 30fb26289399bea925b8ee00a5000f321dcbab3d551c53e62737a03f27d7f75e (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-31T02:49:55.662 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 podman[113730]: 2024-08-31 02:49:55.439043367 +0000 UTC m=+0.015590932 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:49:55.662 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 podman[113730]: 2024-08-31 02:49:55.540136009 +0000 UTC m=+0.116683605 container create c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-31T02:49:55.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:55 smithi070 ceph-mon[104591]: pgmap v98: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:55.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:55 smithi070 ceph-mon[108413]: pgmap v98: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:55.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 podman[113730]: 2024-08-31 02:49:55.792696279 +0000 UTC m=+0.369243833 container init c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-31T02:49:55.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 podman[113730]: 2024-08-31 02:49:55.796216749 +0000 UTC m=+0.372764312 container start c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-08-31T02:49:55.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 ceph-osd[113750]: -- 172.21.15.70:0/2835365199 <== mon.0 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e5acdc3a40 con 0x55e5acda1c00 2024-08-31T02:49:55.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 bash[113730]: c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 2024-08-31T02:49:55.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:55 smithi070 systemd[1]: Started Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:49:56.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:55 smithi102 ceph-mon[81198]: pgmap v98: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 100 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:49:57.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v99: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 557 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v100: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v101: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v102: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v103: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v99: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 557 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v100: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v101: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v102: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v103: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v104: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v105: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v106: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v107: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v108: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v104: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v105: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v106: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v107: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v108: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v109: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v110: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v111: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v112: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v113: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v114: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v115: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: pgmap v116: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v109: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v110: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v111: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v112: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v113: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v114: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v115: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: pgmap v116: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:56 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v99: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 557 B/s rd, 0 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v100: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v101: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v102: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v103: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v104: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v105: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v106: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v107: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v108: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v109: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v110: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v111: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v112: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v113: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v114: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:57.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:57.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v115: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: pgmap v116: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:57.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:57.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:56 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:57.699 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:49:57.446+0000 7f0c2482d640 -1 Falling back to public interface 2024-08-31T02:49:58.149 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:49:58.149 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.149 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v117: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v118: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.150 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v119: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v120: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v121: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:49:58.151 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v117: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v118: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.152 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v119: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v120: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v121: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v122: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v123: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v124: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v125: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v126: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.155 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v122: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v123: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v124: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v125: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v126: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v127: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v128: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.158 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v129: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v130: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.159 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v131: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v132: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v133: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: pgmap v134: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.160 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v127: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v128: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.161 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v129: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v130: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v131: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v132: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.162 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v133: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: pgmap v134: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.163 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:57 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:49:58.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v117: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v118: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.320 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v119: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v120: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.321 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v121: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v122: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v123: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.322 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v124: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v125: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v126: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.323 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v127: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v128: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v129: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.324 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v130: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v131: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v132: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:58.325 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:58.326 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:58.326 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:58.326 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v133: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.326 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: pgmap v134: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:58.326 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:57 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v135: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v136: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v137: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v138: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v139: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v140: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v141: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v142: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v143: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v144: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v145: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v146: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v147: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v148: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v149: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: pgmap v150: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:58 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v135: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v136: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v137: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v138: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v139: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v135: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v136: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v137: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v138: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.203 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v139: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v140: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v141: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v142: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v143: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v144: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v145: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v146: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v147: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v148: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v149: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: pgmap v150: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v140: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v141: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v142: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v143: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v144: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v145: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v146: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v147: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v148: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:49:59.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:49:59.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v149: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: pgmap v150: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:49:59.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:49:59.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:58 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.252 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v151: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v152: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v153: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v154: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.253 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v155: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v156: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.254 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v157: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v158: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v159: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v160: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.255 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.256 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v151: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v152: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.257 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v153: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v154: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v155: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v156: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.258 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v157: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v158: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.259 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v159: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v160: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v161: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v162: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.260 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v163: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v164: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.261 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v165: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: pgmap v166: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v161: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v162: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.262 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v163: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v164: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.263 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v165: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.264 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: pgmap v166: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.264 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.264 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:49:59 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v151: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v152: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v153: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v154: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v155: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v156: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v157: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v158: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v159: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v160: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v161: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v162: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v163: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v164: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:00.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:00.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:00.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v165: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: pgmap v166: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:00.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:00.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:49:59 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.199 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:00.755+0000 7f0c2482d640 -1 osd.0 0 read_superblock omap replica is missing. 2024-08-31T02:50:01.199 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:00.779+0000 7f0c2482d640 -1 osd.0 115 log_to_monitors true 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v167: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v168: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in error state 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: [WRN] OSD_DOWN: 1 osds down 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: osd.0 (root=default,host=smithi070) is down 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-08-31T02:50:01.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v169: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v170: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v171: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v172: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v173: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v167: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v168: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in error state 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: [WRN] OSD_DOWN: 1 osds down 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: osd.0 (root=default,host=smithi070) is down 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-08-31T02:50:01.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v174: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v175: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v176: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v177: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v178: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v169: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v170: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v171: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v172: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v173: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v179: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v180: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v181: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v182: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v183: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: pgmap v184: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v174: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v175: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v176: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.214 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.215 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v177: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v178: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v179: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v180: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.216 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v181: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v182: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.217 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v183: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: pgmap v184: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.218 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:00 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v167: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v168: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: daemon iscsi.foo.smithi070.ashysy on smithi070 is in error state 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: [WRN] OSD_DOWN: 1 osds down 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: osd.0 (root=default,host=smithi070) is down 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: [WRN] PG_DEGRADED: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 2.f is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-08-31T02:50:01.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.b is active+undersized+degraded, acting [3,4] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.f is active+undersized+degraded, acting [7,4] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pg 4.3 is active+undersized+degraded, acting [5,7] 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v169: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v170: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v171: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v172: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v173: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v174: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v175: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v176: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v177: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v178: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v179: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v180: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v181: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v182: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v183: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: pgmap v184: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:01.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:00 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:01.949 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:01.798+0000 7f0c1c5d8640 -1 osd.0 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:50:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v185: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v186: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v187: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v188: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v189: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v190: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v191: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v192: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v193: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v194: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v195: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v196: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v197: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v198: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: osdmap e118: 8 total, 7 up, 8 in 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:02.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v200: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: pgmap v201: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:01 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v185: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v186: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v187: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v188: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v189: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.451 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v185: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v186: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v187: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v188: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v189: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v190: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v191: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v192: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v193: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v194: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v190: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v191: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v192: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v193: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v194: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v195: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v196: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v197: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v198: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: osdmap e118: 8 total, 7 up, 8 in 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v200: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: pgmap v201: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v195: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v196: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v197: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v198: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 101 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: osdmap e118: 8 total, 7 up, 8 in 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v200: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: pgmap v201: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:02.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:02.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:02.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:02.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:01 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v202: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v203: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v204: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.178 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v205: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v206: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v207: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.179 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v202: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v203: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.180 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v204: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v205: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.181 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v206: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v207: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.182 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v208: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v209: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v210: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v211: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.183 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v212: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v213: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.184 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v208: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v209: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v210: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.185 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v211: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v212: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v213: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.186 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v214: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v215: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.187 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v216: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v217: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.188 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552] boot 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: osdmap e119: 8 total, 8 up, 8 in 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v219: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: pgmap v220: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.189 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v214: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v215: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.190 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v216: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v217: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552] boot 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: osdmap e119: 8 total, 8 up, 8 in 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.191 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v219: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: pgmap v220: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.192 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:02 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v202: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v203: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v204: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v205: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v206: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v207: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v208: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v209: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v210: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v211: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v212: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v213: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v214: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v215: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v216: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v217: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: osd.0 [v2:172.21.15.70:6800/852335552,v1:172.21.15.70:6801/852335552] boot 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: osdmap e119: 8 total, 8 up, 8 in 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v219: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: pgmap v220: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:03.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:03.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:02 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:03.973 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:03] "GET /metrics HTTP/1.1" 200 38138 "" "Prometheus/2.43.0" 2024-08-31T02:50:04.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v221: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v222: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v223: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v224: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v225: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v221: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v222: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v223: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v224: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v225: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v226: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v227: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v228: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v229: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v230: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v226: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v227: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v228: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v229: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v230: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v231: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v232: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v233: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v234: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: osdmap e120: 8 total, 8 up, 8 in 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v236: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: pgmap v237: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v231: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v232: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v233: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v234: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: osdmap e120: 8 total, 8 up, 8 in 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v236: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: pgmap v237: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v221: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v222: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v223: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v224: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v225: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v226: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v227: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v228: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v229: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v230: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v231: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v232: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.427 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v233: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v234: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: osdmap e120: 8 total, 8 up, 8 in 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.428 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v236: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: pgmap v237: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:04.429 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.430 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:04.430 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:04.430 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:04.430 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:04.430 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v238: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v239: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v240: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.348 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v241: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v242: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v243: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.349 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v238: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v239: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v240: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v241: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v242: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v243: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v244: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v245: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.352 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v246: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v247: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v248: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.353 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v249: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v244: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v245: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.354 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v246: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v247: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v248: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.355 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v249: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v250: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v251: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v252: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v253: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v254: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: pgmap v255: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v250: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v251: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v252: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v253: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v254: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: pgmap v255: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:04 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v238: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v239: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v240: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v241: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v242: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v243: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v244: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v245: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v246: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v247: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v248: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v249: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v250: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v251: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v252: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v253: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v254: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: pgmap v255: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:05.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:04 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v256: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v257: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v258: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v259: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v260: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v256: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v257: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v258: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v259: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v260: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v261: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.204 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v262: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v263: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v264: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v265: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v266: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v267: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v268: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v269: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v270: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v271: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v272: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: pgmap v273: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v261: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v262: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v263: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v264: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v265: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v266: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v267: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v268: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v269: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v270: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v271: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v272: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: pgmap v273: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:05 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.238 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v256: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v257: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.239 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v258: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v259: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.240 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v260: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v261: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.241 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v262: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v263: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v264: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v265: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v266: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v267: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v268: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v269: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v270: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v271: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.244 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v272: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: pgmap v273: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:06.245 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:05 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.316 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v274: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v275: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.317 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v276: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v277: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.318 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v278: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.319 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v279: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v280: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v281: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.320 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v282: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v283: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.321 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.322 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v274: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v275: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v276: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v277: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.323 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.324 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v278: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v284: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v285: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.325 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v286: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v287: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.326 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.327 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.328 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v279: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.328 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v280: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v281: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.329 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v282: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v283: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.330 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v284: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v285: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.331 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v286: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v287: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.332 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.333 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:06 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:07.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v274: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v275: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v276: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v277: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v278: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v279: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v280: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v281: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v282: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v283: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v284: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v285: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v286: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v287: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 32/723 objects degraded (4.426%) 2024-08-31T02:50:07.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:07.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:07.426 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:06 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.175 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.176 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.177 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:07 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:08.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:07 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.170 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.171 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.172 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.173 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.174 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:09 smithi102 ceph-mon[81198]: Upgrade: osd.0 is safe to restart 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.205 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.206 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.207 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.208 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.209 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.210 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[108413]: Upgrade: osd.0 is safe to restart 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.211 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-31T02:50:09.212 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:09 smithi070 ceph-mon[104591]: Upgrade: osd.0 is safe to restart 2024-08-31T02:50:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:11 smithi102 ceph-mon[81198]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:50:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:11 smithi102 ceph-mon[81198]: Upgrade: Updating osd.0 2024-08-31T02:50:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:50:12.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:11 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:12.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:11 smithi102 ceph-mon[81198]: Deploying daemon osd.0 on smithi070 2024-08-31T02:50:12.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[104591]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[104591]: Upgrade: Updating osd.0 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[104591]: Deploying daemon osd.0 on smithi070 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[108413]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[108413]: Upgrade: Updating osd.0 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:12.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:11 smithi070 ceph-mon[108413]: Deploying daemon osd.0 on smithi070 2024-08-31T02:50:13.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:13.951 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:13 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:13] "GET /metrics HTTP/1.1" 200 38138 "" "Prometheus/2.43.0" 2024-08-31T02:50:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:14.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:13 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:15.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:14 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: pgmap v329: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:15.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: pgmap v330: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:15.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:15.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:15.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:14 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:17.088 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:16 smithi070 ceph-mon[104591]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:17.088 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:16 smithi070 ceph-mon[108413]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:17.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:16 smithi102 ceph-mon[81198]: pgmap v332: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:18.106 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:17 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:18.106 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:17 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:18.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:17 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:18.590 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:50:18.699 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:18 smithi070 systemd[1]: Stopping Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:50:19.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:18 smithi102 ceph-mon[81198]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:19.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:18 smithi070 ceph-mon[104591]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:19.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:18 smithi070 ceph-mon[108413]: pgmap v333: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:50:19.200 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:18.938+0000 7f0c217c5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:50:19.200 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:18.938+0000 7f0c217c5640 -1 osd.0 120 *** Got signal Terminated *** 2024-08-31T02:50:19.200 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:18 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[113746]: 2024-08-31T02:50:18.938+0000 7f0c217c5640 -1 osd.0 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 5s ago 35m - - 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 11s ago 35m - - 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (5m) 5s ago 30m 26.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (5m) 11s ago 30m 79.3M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 error 5s ago 29m 63.8M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (8m) 11s ago 35m 507M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:50:19.301 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (19m) 5s ago 36m 439M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (2m) 5s ago 36m 59.4M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (3m) 11s ago 35m 45.1M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (2m) 5s ago 35m 41.1M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 7a529ea3924f 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (7m) 5s ago 31m 16.0M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (7m) 11s ago 31m 16.3M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (23s) 5s ago 34m 43.3M 4096M 19.1.1-240-g34b9d27b 72d622302cc2 c70c04c6d179 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (34m) 5s ago 34m 59.7M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (34m) 5s ago 34m 52.7M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (33m) 5s ago 33m 55.2M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (33m) 11s ago 33m 57.6M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:50:19.302 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (32m) 11s ago 32m 54.1M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:50:19.303 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (32m) 11s ago 32m 55.5M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:50:19.303 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (32m) 11s ago 32m 59.5M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:50:19.303 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 11s ago 31m 55.6M - 2.43.0 a07b618ecd1d 8c3920246ae0 2024-08-31T02:50:19.303 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (29m) 5s ago 29m 80.7M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:50:19.303 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (29m) 11s ago 29m 79.6M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:50:19.701 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:50:19.701 INFO:teuthology.orchestra.run.smithi070.stdout: "mon": { 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 3 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "mgr": { 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 2 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "osd": { 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 1 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "rgw": { 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "overall": { 2024-08-31T02:50:19.702 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-08-31T02:50:19.703 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph version 19.1.1-240-g34b9d27b (34b9d27bc925f169ace27b9f1f204536aa49fc0b) squid (rc)": 6 2024-08-31T02:50:19.703 INFO:teuthology.orchestra.run.smithi070.stdout: } 2024-08-31T02:50:19.703 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769", 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "in_progress": true, 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "services_complete": [], 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "progress": "1/8 daemons upgraded", 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "message": "Currently upgrading osd daemons", 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout: "is_paused": false 2024-08-31T02:50:20.044 INFO:teuthology.orchestra.run.smithi070.stdout:} 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[104591]: from='client.64311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[104591]: osd.0 marked itself down and dead 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/1075056943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[108413]: from='client.64311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[108413]: osd.0 marked itself down and dead 2024-08-31T02:50:20.045 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:19 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/1075056943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:19 smithi102 ceph-mon[81198]: from='client.64311 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:19 smithi102 ceph-mon[81198]: osd.0 marked itself down and dead 2024-08-31T02:50:20.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:19 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/1075056943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:20.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:20 smithi070 podman[119468]: 2024-08-31 02:50:20.292538105 +0000 UTC m=+1.474063430 container died c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-08-31T02:50:20.789 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:20 smithi070 podman[119468]: 2024-08-31 02:50:20.680647512 +0000 UTC m=+1.862172865 container cleanup c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-08-31T02:50:20.790 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:20 smithi070 bash[119468]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0 2024-08-31T02:50:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:20 smithi102 ceph-mon[81198]: from='client.64317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:20 smithi102 ceph-mon[81198]: from='client.64323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:20 smithi102 ceph-mon[81198]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-08-31T02:50:21.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:20 smithi102 ceph-mon[81198]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:50:21.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:20 smithi102 ceph-mon[81198]: osdmap e121: 8 total, 7 up, 8 in 2024-08-31T02:50:21.199 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:20 smithi070 podman[119554]: 2024-08-31 02:50:20.990672761 +0000 UTC m=+0.692169578 container remove c70c04c6d179359f3fa3f70d4de15ed8ab6fa3fdbb81263fe3aa51c78411aba7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-34b9d27, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-31T02:50:21.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[108413]: from='client.64317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.199 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[108413]: from='client.64323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[108413]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[108413]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[108413]: osdmap e121: 8 total, 7 up, 8 in 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[104591]: from='client.64317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[104591]: from='client.64323 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[104591]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 777 B/s rd, 0 op/s 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[104591]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-31T02:50:21.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:20 smithi070 ceph-mon[104591]: osdmap e121: 8 total, 7 up, 8 in 2024-08-31T02:50:21.661 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.246175727 +0000 UTC m=+0.016119095 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:50:21.661 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.350307198 +0000 UTC m=+0.120250598 container create 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27) 2024-08-31T02:50:21.661 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.577824305 +0000 UTC m=+0.347767665 container init 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2) 2024-08-31T02:50:21.661 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.581288078 +0000 UTC m=+0.351231433 container start 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-08-31T02:50:21.661 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.660866316 +0000 UTC m=+0.430809673 container attach 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-08-31T02:50:21.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:21 smithi070 ceph-mon[104591]: from='client.64335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.950 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:21 smithi070 ceph-mon[104591]: osdmap e122: 8 total, 7 up, 8 in 2024-08-31T02:50:21.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:21 smithi070 ceph-mon[108413]: from='client.64335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:21.950 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:21 smithi070 ceph-mon[108413]: osdmap e122: 8 total, 7 up, 8 in 2024-08-31T02:50:21.951 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:21 smithi070 podman[119644]: 2024-08-31 02:50:21.766422858 +0000 UTC m=+0.536366223 container died 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-31T02:50:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:21 smithi102 ceph-mon[81198]: from='client.64335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:22.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:21 smithi102 ceph-mon[81198]: osdmap e122: 8 total, 7 up, 8 in 2024-08-31T02:50:22.699 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 podman[119668]: 2024-08-31 02:50:22.423408292 +0000 UTC m=+0.650922424 container remove 9f79f9bb927c2a2aaff37037cc6df5d2e93e0452c00f31fecf6b9279e0720c51 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-31T02:50:22.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-31T02:50:22.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 systemd[1]: Stopped Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:50:22.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service: Consumed 1.758s CPU time. 2024-08-31T02:50:22.700 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 systemd[1]: Starting Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:50:23.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:22 smithi070 ceph-mon[104591]: pgmap v337: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:23.153 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:22 smithi070 ceph-mon[104591]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-08-31T02:50:23.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:22 smithi070 ceph-mon[108413]: pgmap v337: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:23.153 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:22 smithi070 ceph-mon[108413]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-08-31T02:50:23.153 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 podman[119768]: 2024-08-31 02:50:22.762789322 +0000 UTC m=+0.015886517 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:50:23.154 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:22 smithi070 podman[119768]: 2024-08-31 02:50:22.89994643 +0000 UTC m=+0.153043621 container create 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-31T02:50:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:22 smithi102 ceph-mon[81198]: pgmap v337: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:23.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:22 smithi102 ceph-mon[81198]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 podman[119768]: 2024-08-31 02:50:23.152531865 +0000 UTC m=+0.405629055 container init 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 podman[119768]: 2024-08-31 02:50:23.156099813 +0000 UTC m=+0.409197010 container start 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 podman[119768]: 2024-08-31 02:50:23.244176832 +0000 UTC m=+0.497274026 container attach 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.450 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-31T02:50:23.451 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-31T02:50:23.950 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate[119783]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-31T02:50:23.951 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 bash[119768]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-31T02:50:23.951 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:23 smithi070 podman[119768]: 2024-08-31 02:50:23.701525533 +0000 UTC m=+0.954622723 container died 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-31T02:50:24.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:23 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:23] "GET /metrics HTTP/1.1" 200 38150 "" "Prometheus/2.43.0" 2024-08-31T02:50:24.802 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:24 smithi070 podman[119906]: 2024-08-31 02:50:24.523586358 +0000 UTC m=+0.816020430 container remove 3bab7f55f34cce96ac8b333104d82c16a43daef32df624bbc8e6427294853698 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-08-31T02:50:24.802 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:24 smithi070 podman[119967]: 2024-08-31 02:50:24.632528337 +0000 UTC m=+0.015717721 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:50:24.802 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:24 smithi070 podman[119967]: 2024-08-31 02:50:24.766307822 +0000 UTC m=+0.149497199 container create 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-08-31T02:50:25.085 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:24 smithi070 ceph-mon[104591]: pgmap v338: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:50:25.085 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:24 smithi070 ceph-mon[108413]: pgmap v338: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:50:25.086 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:24 smithi070 podman[119967]: 2024-08-31 02:50:24.98535712 +0000 UTC m=+0.368546496 container init 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=squid-34b9d27, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-31T02:50:25.086 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:24 smithi070 podman[119967]: 2024-08-31 02:50:24.988463749 +0000 UTC m=+0.371653126 container start 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2) 2024-08-31T02:50:25.086 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:25 smithi070 ceph-osd[119986]: -- 172.21.15.70:0/2902728826 <== mon.1 v2:172.21.15.70:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55be1903ba40 con 0x55be19e02000 2024-08-31T02:50:25.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:24 smithi102 ceph-mon[81198]: pgmap v338: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-08-31T02:50:25.431 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:25 smithi070 bash[119967]: 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 2024-08-31T02:50:25.431 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:25 smithi070 systemd[1]: Started Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:50:26.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v339: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 674 B/s rd, 0 op/s 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v340: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v341: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v342: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v343: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v344: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v345: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v346: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v347: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v348: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v349: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v350: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v351: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: pgmap v352: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:26 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v339: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 674 B/s rd, 0 op/s 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v340: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v341: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v342: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v343: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v339: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 674 B/s rd, 0 op/s 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v340: 161 pgs: 58 peering, 103 active+clean; 585 KiB data, 117 MiB used, 715 GiB / 715 GiB avail; 914 B/s rd, 0 op/s 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.452 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v341: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v342: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.453 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v343: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v344: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v345: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v346: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v347: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v348: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v344: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 2 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.456 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v345: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v346: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v347: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.457 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v348: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v349: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v350: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v351: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: pgmap v352: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v349: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v350: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v351: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: pgmap v352: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:26.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.137 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:26 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:50:26.868+0000 7f4be7f6c640 -1 Falling back to public interface 2024-08-31T02:50:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v353: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v354: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v355: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v356: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v357: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v358: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v359: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v360: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v361: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v362: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v363: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v364: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v365: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v366: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v367: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: pgmap v368: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:27 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v353: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v354: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v355: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v356: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-08-31T02:50:27.425 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v357: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v358: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.426 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v359: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v360: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v353: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v354: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v355: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v356: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Health check failed: Degraded data redundancy: 76/723 objects degraded (10.512%), 19 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:27.427 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 9 pgs peering) 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v357: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v358: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.428 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v359: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v360: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v361: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v362: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v363: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v364: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v361: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v362: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v363: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v364: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v365: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v366: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v367: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: pgmap v368: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.434 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.435 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v365: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v366: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v367: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: pgmap v368: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:27.437 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:27.438 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:27 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v369: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v370: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v371: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v372: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v373: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v374: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.201 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v375: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v376: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.202 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v369: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v370: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v371: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v372: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v373: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v374: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.203 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v375: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v376: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.204 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v377: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v378: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v379: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v380: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.205 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v381: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.206 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v382: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v377: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v378: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.207 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v379: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v380: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v381: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v382: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.208 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v383: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v384: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.209 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v385: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: pgmap v386: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.210 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v383: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v384: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.211 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v385: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: pgmap v386: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.212 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.213 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:28 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v369: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v370: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v371: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v372: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v373: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v374: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v375: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v376: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.420 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v377: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v378: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.421 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v379: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v380: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.422 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v381: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v382: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.423 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v383: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v384: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v385: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: pgmap v386: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:28.424 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:28.425 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:28 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v387: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v388: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v389: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v390: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.154 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.155 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v387: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.155 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v388: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v389: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v390: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v391: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v392: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.156 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v393: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v394: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v395: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v396: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.157 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.158 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.273 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v387: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v388: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v389: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v390: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v391: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v392: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.274 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v393: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v394: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.275 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v395: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v396: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.276 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v397: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v398: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v399: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v400: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.277 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v401: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v402: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.278 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v403: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: pgmap v404: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.279 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.280 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.280 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.280 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:29 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v391: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v392: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.413 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v393: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v394: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v395: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v396: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.414 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v397: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v398: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v399: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v400: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v401: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v402: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.416 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v403: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: pgmap v404: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.417 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.418 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v397: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v398: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.419 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v399: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v400: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v401: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v402: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.420 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v403: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: pgmap v404: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.421 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:29.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:29 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.314 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v405: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.314 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v406: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.314 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.314 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v407: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v408: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v409: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v410: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.315 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v411: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v412: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.316 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v413: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v414: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v415: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v416: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.317 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v417: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v418: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.318 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v419: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: pgmap v420: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.319 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:30 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v405: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v406: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v407: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v408: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.439 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v409: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v410: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v411: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.440 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v412: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v405: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v406: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v407: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v408: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.441 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v409: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v410: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.442 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v411: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v412: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v413: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.443 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v414: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v415: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v416: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.444 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v417: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.445 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v413: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v414: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v415: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v416: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.446 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v417: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v418: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.447 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v419: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: pgmap v420: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.448 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v418: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v419: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: pgmap v420: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:30.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:30.701 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:50:30.437+0000 7f4be7f6c640 -1 osd.0 120 log_to_monitors true 2024-08-31T02:50:31.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v421: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.422 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v422: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v423: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v424: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.423 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v425: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v426: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.424 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v421: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v422: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v423: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v424: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.425 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v425: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v426: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.426 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v427: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v428: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.427 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v429: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v430: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.428 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v431: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v432: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v433: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v434: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.429 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v435: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: pgmap v436: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.430 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v427: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v428: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.431 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v429: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v430: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.432 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v431: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v432: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.433 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v433: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v434: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.434 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v435: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: pgmap v436: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.435 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.436 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v421: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v422: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v423: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v424: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v425: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v426: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v427: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v428: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 13 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v429: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v430: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 14 op/s; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v431: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v432: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v433: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v434: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v435: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: pgmap v436: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:31.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:31.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:31.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:31.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:31 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:31.675 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:50:31 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:50:31.421+0000 7f4bdfd17640 -1 osd.0 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v437: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v438: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v439: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v440: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v441: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v442: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v443: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v444: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: osdmap e123: 8 total, 7 up, 8 in 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v446: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v447: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v448: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v449: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v450: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v451: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v452: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v453: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v454: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: pgmap v455: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v437: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v438: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v439: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v440: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v441: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v442: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v443: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v444: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: osdmap e123: 8 total, 7 up, 8 in 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v446: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v447: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v448: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v449: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v450: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v451: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v452: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v453: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v454: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: pgmap v455: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.465 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:32 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v437: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v438: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v439: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v440: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v441: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v442: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v443: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v444: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: osdmap e123: 8 total, 7 up, 8 in 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v446: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v447: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v448: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v449: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v450: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v451: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v452: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v453: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v454: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: pgmap v455: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:32.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:32 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v456: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.451 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v457: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v458: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v459: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v460: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v461: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.452 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v462: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v463: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529] boot 2024-08-31T02:50:33.453 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: osdmap e124: 8 total, 8 up, 8 in 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v465: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v466: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.454 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v467: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v468: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v469: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v470: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.455 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v471: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: pgmap v472: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.456 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.457 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.458 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v456: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v457: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.458 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v458: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v459: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v460: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v461: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.459 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v462: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v463: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529] boot 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: osdmap e124: 8 total, 8 up, 8 in 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.460 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v465: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v466: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v467: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.461 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v468: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v469: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v470: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.462 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v471: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: pgmap v472: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.463 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.464 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:33 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v456: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v457: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v458: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v459: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v460: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v461: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v462: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v463: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: osd.0 [v2:172.21.15.70:6800/3227414529,v1:172.21.15.70:6801/3227414529] boot 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: osdmap e124: 8 total, 8 up, 8 in 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v465: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v466: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v467: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v468: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v469: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v470: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v471: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: pgmap v472: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:33.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:33.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:33 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:33] "GET /metrics HTTP/1.1" 200 38163 "" "Prometheus/2.43.0" 2024-08-31T02:50:34.349 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v473: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.349 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v474: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.349 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.349 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v475: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.349 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v476: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v477: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v478: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.350 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v479: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v480: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: osdmap e125: 8 total, 8 up, 8 in 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.351 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v482: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v483: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.352 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v484: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v485: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v486: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v487: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.353 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v473: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v474: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v475: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v476: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.354 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v477: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v478: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.355 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v479: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v480: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.356 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.357 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: osdmap e125: 8 total, 8 up, 8 in 2024-08-31T02:50:34.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:50:34.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.357 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Upgrade: Finalizing container_image settings 2024-08-31T02:50:34.358 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:50:34.359 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:50:34.360 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:50:34.361 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.361 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.361 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v482: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v483: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.362 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v484: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v485: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.363 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v486: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v487: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.364 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: Upgrade: Complete! 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v488: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v489: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.365 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v490: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: pgmap v491: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.366 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Upgrade: Finalizing container_image settings 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:50:34.367 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.368 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: Upgrade: Complete! 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v488: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v489: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.369 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v490: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: pgmap v491: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.370 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:34 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v473: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v474: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v475: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v476: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v477: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v478: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.669 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v479: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v480: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.670 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: osdmap e125: 8 total, 8 up, 8 in 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.671 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v482: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v483: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v484: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v485: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Metadata not up to date on all hosts. Skipping non agent specs 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.672 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v486: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v487: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Upgrade: Setting container_image for all nvmeof 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.673 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Upgrade: Finalizing container_image settings 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-31T02:50:34.674 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: Upgrade: Complete! 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-31T02:50:34.675 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v488: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v489: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v490: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: pgmap v491: 161 pgs: 39 active+undersized, 19 active+undersized+degraded, 103 active+clean; 585 KiB data, 118 MiB used, 715 GiB / 715 GiB avail; 76/723 objects degraded (10.512%) 2024-08-31T02:50:34.676 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:34 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:35.522 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:35 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:35.522 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:35 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:35.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:35 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:36.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:36 smithi102 ceph-mon[81198]: pgmap v492: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:36.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:36 smithi070 ceph-mon[104591]: pgmap v492: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:36.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:36 smithi070 ceph-mon[108413]: pgmap v492: 161 pgs: 14 active+undersized, 7 active+undersized+degraded, 140 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s; 32/723 objects degraded (4.426%) 2024-08-31T02:50:37.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:37 smithi070 ceph-mon[104591]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:37.526 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:37 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:37.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:37 smithi070 ceph-mon[108413]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:37.526 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:37 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:37 smithi102 ceph-mon[81198]: Health check update: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded (PG_DEGRADED) 2024-08-31T02:50:37.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:37 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:38.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:38 smithi070 ceph-mon[104591]: pgmap v493: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:38.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:38 smithi070 ceph-mon[108413]: pgmap v493: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:38.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:38 smithi102 ceph-mon[81198]: pgmap v493: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:39.523 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:39 smithi070 ceph-mon[104591]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:39.523 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:39 smithi070 ceph-mon[108413]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:39.774 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:39 smithi102 ceph-mon[81198]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32/723 objects degraded (4.426%), 7 pgs degraded) 2024-08-31T02:50:40.584 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:40 smithi070 ceph-mon[104591]: pgmap v494: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-08-31T02:50:40.584 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:40 smithi070 ceph-mon[108413]: pgmap v494: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-08-31T02:50:40.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:40 smithi102 ceph-mon[81198]: pgmap v494: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 837 B/s rd, 0 op/s 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: pgmap v495: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: pgmap v496: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.169 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:42 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: pgmap v495: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: pgmap v496: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: pgmap v495: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: pgmap v496: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:50:43.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:43.201 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:42 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:44.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:43 smithi102 ceph-mon[81198]: pgmap v497: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:44.168 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:43 smithi102 ceph-mon[81198]: pgmap v498: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:44.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:43] "GET /metrics HTTP/1.1" 200 38163 "" "Prometheus/2.43.0" 2024-08-31T02:50:44.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:43 smithi070 ceph-mon[104591]: pgmap v497: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:44.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:43 smithi070 ceph-mon[104591]: pgmap v498: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:43 smithi070 ceph-mon[108413]: pgmap v497: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:44.200 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:43 smithi070 ceph-mon[108413]: pgmap v498: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:45 smithi102 ceph-mon[81198]: pgmap v499: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:45.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:45 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:45.699 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[108413]: pgmap v499: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[104591]: pgmap v499: 161 pgs: 161 active+clean; 585 KiB data, 123 MiB used, 715 GiB / 715 GiB avail; 840 B/s rd, 0 op/s 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:50:45.700 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:45 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:47 smithi102 ceph-mon[81198]: pgmap v500: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 0 B/s wr, 197 op/s 2024-08-31T02:50:47.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:47 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:47 smithi070 ceph-mon[104591]: pgmap v500: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 0 B/s wr, 197 op/s 2024-08-31T02:50:47.699 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:47 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:47 smithi070 ceph-mon[108413]: pgmap v500: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 0 B/s wr, 197 op/s 2024-08-31T02:50:47.700 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:47 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: pgmap v501: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 230 op/s 2024-08-31T02:50:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: pgmap v502: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 223 op/s 2024-08-31T02:50:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: pgmap v503: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 231 op/s 2024-08-31T02:50:49.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:49 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: pgmap v501: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 230 op/s 2024-08-31T02:50:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: pgmap v502: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 223 op/s 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: pgmap v503: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 231 op/s 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: pgmap v501: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 230 op/s 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: pgmap v502: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 0 B/s wr, 223 op/s 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: pgmap v503: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 0 B/s wr, 231 op/s 2024-08-31T02:50:49.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:49 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:50:51.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch ps' 2024-08-31T02:50:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:51 smithi070 ceph-mon[104591]: pgmap v504: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s 2024-08-31T02:50:51.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:51 smithi070 ceph-mon[104591]: from='client.64347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:51 smithi070 ceph-mon[108413]: pgmap v504: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s 2024-08-31T02:50:51.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:51 smithi070 ceph-mon[108413]: from='client.64347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:51.465 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:51 smithi102 ceph-mon[81198]: pgmap v504: 161 pgs: 161 active+clean; 585 KiB data, 128 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s 2024-08-31T02:50:51.465 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:51 smithi102 ceph-mon[81198]: from='client.64347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:53.412 INFO:teuthology.orchestra.run.smithi070.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi070 smithi070 running 11s ago 36m - - 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:agent.smithi102 smithi102 running 5s ago 36m - - 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:alertmanager.a smithi070 *:9093,9094 running (6m) 11s ago 31m 26.9M - 0.25.0 c8568f914cd2 df358a3d49e7 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:grafana.a smithi102 *:3000 running (5m) 5s ago 31m 79.9M - 9.4.12 2bacad6d85d8 841fb84c461e 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:iscsi.foo.smithi070.ashysy smithi070 error 11s ago 29m 63.8M - 3.5 e1d6a67b021e 22d8704fdce3 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.x smithi102 *:8443,9283,8765 running (8m) 5s ago 35m 511M - 19.1.1-240-g34b9d27b 72d622302cc2 d17af6ad69a7 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:mgr.y smithi070 *:8443,9283,8765 running (19m) 11s ago 37m 440M - 19.1.1-240-g34b9d27b 72d622302cc2 01db10935c21 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:mon.a smithi070 running (3m) 11s ago 37m 65.2M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 616367ace055 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:mon.b smithi102 running (4m) 5s ago 35m 47.7M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 614209a50434 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:mon.c smithi070 running (2m) 11s ago 36m 43.0M 2048M 19.1.1-240-g34b9d27b 72d622302cc2 7a529ea3924f 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.a smithi070 *:9100 running (7m) 11s ago 31m 16.2M - 1.5.0 0da6a335fe13 95e4443a43cb 2024-08-31T02:50:53.413 INFO:teuthology.orchestra.run.smithi070.stdout:node-exporter.b smithi102 *:9100 running (7m) 5s ago 31m 16.0M - 1.5.0 0da6a335fe13 f63aad055833 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.0 smithi070 running (28s) 11s ago 35m 41.4M 4096M 19.1.1-240-g34b9d27b 72d622302cc2 23c731c518c0 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.1 smithi070 running (35m) 11s ago 35m 60.2M 4096M 17.2.0 e1d6a67b021e 2ad147a0039c 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.2 smithi070 running (34m) 11s ago 34m 52.7M 4096M 17.2.0 e1d6a67b021e f13f7407a65c 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.3 smithi070 running (34m) 11s ago 34m 55.6M 4096M 17.2.0 e1d6a67b021e d13f8745f80e 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.4 smithi102 running (33m) 5s ago 33m 59.5M 4096M 17.2.0 e1d6a67b021e 430432c25429 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.5 smithi102 running (33m) 5s ago 33m 55.6M 4096M 17.2.0 e1d6a67b021e a2668180d986 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.6 smithi102 running (33m) 5s ago 33m 57.5M 4096M 17.2.0 e1d6a67b021e d4eb46bd6282 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:osd.7 smithi102 running (32m) 5s ago 32m 61.2M 4096M 17.2.0 e1d6a67b021e 72b7b2af68b0 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:prometheus.a smithi102 *:9095 running (2m) 5s ago 31m 56.1M - 2.43.0 a07b618ecd1d 8c3920246ae0 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi070.usqphe smithi070 *:8000 running (30m) 11s ago 30m 80.8M - 17.2.0 e1d6a67b021e f0ee575be8b8 2024-08-31T02:50:53.414 INFO:teuthology.orchestra.run.smithi070.stdout:rgw.foo.smithi102.axbapt smithi102 *:8000 running (30m) 5s ago 30m 80.0M - 17.2.0 e1d6a67b021e 72c4fdae72fa 2024-08-31T02:50:53.415 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:53 smithi070 ceph-mon[104591]: pgmap v505: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 175 op/s 2024-08-31T02:50:53.415 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:53 smithi070 ceph-mon[108413]: pgmap v505: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 175 op/s 2024-08-31T02:50:53.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:53 smithi102 ceph-mon[81198]: pgmap v505: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 0 B/s wr, 175 op/s 2024-08-31T02:50:54.165 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-08-31T02:50:54.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:50:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:50:53] "GET /metrics HTTP/1.1" 200 38152 "" "Prometheus/2.43.0" 2024-08-31T02:50:54.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:54 smithi070 ceph-mon[104591]: from='client.74257 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:54.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:54 smithi070 ceph-mon[108413]: from='client.74257 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:54.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:54 smithi102 ceph-mon[81198]: from='client.74257 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:50:55.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:55 smithi070 ceph-mon[104591]: pgmap v506: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s 2024-08-31T02:50:55.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:55 smithi070 ceph-mon[108413]: pgmap v506: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s 2024-08-31T02:50:55.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:55 smithi102 ceph-mon[81198]: pgmap v506: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 29 op/s 2024-08-31T02:50:56.398 INFO:teuthology.orchestra.run.smithi070.stdout:true 2024-08-31T02:50:57.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-08-31T02:50:57.373 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[104591]: pgmap v507: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:57.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[104591]: from='client.? 172.21.15.70:0/381271754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:57.374 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[104591]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:57.374 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[108413]: pgmap v507: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:57.374 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[108413]: from='client.? 172.21.15.70:0/381271754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:57.374 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:57 smithi070 ceph-mon[108413]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:57 smithi102 ceph-mon[81198]: pgmap v507: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-08-31T02:50:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:57 smithi102 ceph-mon[81198]: from='client.? 172.21.15.70:0/381271754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-31T02:50:57.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:57 smithi102 ceph-mon[81198]: from='client.15468 -' entity='client.iscsi.foo.smithi070.ashysy' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-08-31T02:50:59.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:50:59 smithi070 ceph-mon[104591]: pgmap v508: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:50:59.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:50:59 smithi070 ceph-mon[108413]: pgmap v508: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:50:59.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:50:59 smithi102 ceph-mon[81198]: pgmap v508: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-08-31T02:51:00.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:00 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:51:00.449 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:00 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:51:00.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:00 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-31T02:51:01.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:01 smithi102 ceph-mon[81198]: from='client.74269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:51:01.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:01 smithi102 ceph-mon[81198]: pgmap v509: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:51:01.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:01 smithi070 ceph-mon[104591]: from='client.74269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:51:01.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:01 smithi070 ceph-mon[104591]: pgmap v509: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:51:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:01 smithi070 ceph-mon[108413]: from='client.74269 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:34b9d27bc925f169ace27b9f1f204536aa49fc0b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-31T02:51:01.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:01 smithi070 ceph-mon[108413]: pgmap v509: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-08-31T02:51:02.085 INFO:teuthology.orchestra.run.smithi070.stdout:false 2024-08-31T02:51:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-mon[81198]: pgmap v510: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:51:03.418 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-mon[81198]: pgmap v511: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:51:03.419 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-mon[81198]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.449 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[104591]: pgmap v510: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[104591]: pgmap v511: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[104591]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[108413]: pgmap v510: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[108413]: pgmap v511: 161 pgs: 161 active+clean; 585 KiB data, 129 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-08-31T02:51:03.450 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:03 smithi070 ceph-mon[108413]: from='mgr.54158 172.21.15.102:0/152906120' entity='mgr.x' 2024-08-31T02:51:03.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:51:03.585 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa/tasks/cephadm.py", line 1476, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi070 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c \'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \'"\'"\'.up_to_date | length == 7\'"\'"\'\'' 2024-08-31T02:51:03.827 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b20128395b0341bc9ef0ba1e668a21ad Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa/tasks/cephadm.py", line 1476, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_244a32d3583bb0aa259286ff61157888a54a3ca5/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi070 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c \'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \'"\'"\'.up_to_date | length == 7\'"\'"\'\'' 2024-08-31T02:51:03.830 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-31T02:51:03.839 INFO:tasks.cephadm:Teardown begin 2024-08-31T02:51:03.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:51:03.877 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:51:03.913 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-31T02:51:03.914 DEBUG:teuthology.orchestra.run.smithi070:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-31T02:51:03.935 DEBUG:teuthology.orchestra.run.smithi102:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-31T02:51:03.971 INFO:tasks.cephadm:Stopping all daemons... 2024-08-31T02:51:03.971 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-31T02:51:03.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a 2024-08-31T02:51:04.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: ::ffff:172.21.15.102 - - [31/Aug/2024:02:51:03] "GET /metrics HTTP/1.1" 200 38151 "" "Prometheus/2.43.0" 2024-08-31T02:51:04.199 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 systemd[1]: Stopping Ceph mon.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:04.642 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[104587]: 2024-08-31T02:51:04.291+0000 7ff227fb8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:04.643 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a[104587]: 2024-08-31T02:51:04.291+0000 7ff227fb8640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-31T02:51:04.643 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 podman[124924]: 2024-08-31 02:51:04.322024495 +0000 UTC m=+0.151379683 container died 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-34b9d27, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-31T02:51:04.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE Bus STOPPING 2024-08-31T02:51:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 podman[124924]: 2024-08-31 02:51:04.808284936 +0000 UTC m=+0.637640155 container cleanup 616367ace055cab4beb7226d8c6d5f06395bd88e77c973717864d193117625c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-31T02:51:04.949 INFO:journalctl@ceph.mon.a.smithi070.stdout:Aug 31 02:51:04 smithi070 bash[124924]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-a 2024-08-31T02:51:05.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:51:05.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE Bus STOPPED 2024-08-31T02:51:05.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE Bus STARTING 2024-08-31T02:51:05.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE Serving on http://:::9283 2024-08-31T02:51:05.169 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:04] ENGINE Bus STARTED 2024-08-31T02:51:05.238 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.a.service' 2024-08-31T02:51:05.284 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:05.284 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-31T02:51:05.284 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-31T02:51:05.284 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.c 2024-08-31T02:51:05.613 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:05 smithi070 systemd[1]: Stopping Ceph mon.c for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE Bus STOPPING 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE Bus STOPPED 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE Bus STARTING 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE Serving on http://:::9283 2024-08-31T02:51:05.898 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: [31/Aug/2024:02:51:05] ENGINE Bus STARTED 2024-08-31T02:51:05.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c[108409]: 2024-08-31T02:51:05.612+0000 7fb7699d0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:05.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:05 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c[108409]: 2024-08-31T02:51:05.612+0000 7fb7699d0640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-08-31T02:51:05.949 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:05 smithi070 podman[125046]: 2024-08-31 02:51:05.756025045 +0000 UTC m=+0.241265698 container died 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-31T02:51:06.439 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:06 smithi070 podman[125046]: 2024-08-31 02:51:06.162546846 +0000 UTC m=+0.647787545 container cleanup 7a529ea3924f8f7338c14ae27072772fa4277650db0eb47ab39fd4a5fd17238b (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True) 2024-08-31T02:51:06.439 INFO:journalctl@ceph.mon.c.smithi070.stdout:Aug 31 02:51:06 smithi070 bash[125046]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-c 2024-08-31T02:51:06.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.c.service' 2024-08-31T02:51:06.624 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:06.625 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-31T02:51:06.625 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-31T02:51:06.625 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b 2024-08-31T02:51:06.918 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:06 smithi102 systemd[1]: Stopping Ceph mon.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:07.242 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[81194]: 2024-08-31T02:51:06.957+0000 7f5416942640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:07.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b[81194]: 2024-08-31T02:51:06.957+0000 7f5416942640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-08-31T02:51:07.243 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:06 smithi102 podman[95371]: 2024-08-31 02:51:06.97824353 +0000 UTC m=+0.158387047 container died 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=squid-34b9d27, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-08-31T02:51:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:07 smithi102 podman[95371]: 2024-08-31 02:51:07.433239272 +0000 UTC m=+0.613382834 container cleanup 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, RELEASE=squid-34b9d27, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-08-31T02:51:07.668 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:07 smithi102 bash[95371]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b 2024-08-31T02:51:08.011 INFO:journalctl@ceph.mon.b.smithi102.stdout:Aug 31 02:51:07 smithi102 podman[95396]: 2024-08-31 02:51:07.759855608 +0000 UTC m=+0.774394022 container remove 614209a504343cc7b23ef708d39573398dbb990043dc2919f2de5292757f0c11 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-31T02:51:08.535 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mon.b.service' 2024-08-31T02:51:08.591 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:08.592 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-31T02:51:08.592 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-31T02:51:08.593 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y 2024-08-31T02:51:08.923 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:08 smithi070 systemd[1]: Stopping Ceph mgr.y for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:09.199 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:08 smithi070 podman[125166]: 2024-08-31 02:51:08.923134394 +0000 UTC m=+0.133825734 container died 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-31T02:51:09.561 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:09 smithi070 podman[125166]: 2024-08-31 02:51:09.384212542 +0000 UTC m=+0.594903885 container cleanup 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:51:09.561 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:09 smithi070 bash[125166]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y 2024-08-31T02:51:09.819 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:09 smithi070 podman[125178]: 2024-08-31 02:51:09.560614819 +0000 UTC m=+0.634886840 container remove 01db10935c21e9de6d6d016c9843793eb877ec45de8b10bfe5944c7d9f2f6f59 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-y, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-34b9d27, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-31T02:51:09.819 INFO:journalctl@ceph.mgr.y.smithi070.stdout:Aug 31 02:51:09 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:51:09.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.y.service' 2024-08-31T02:51:09.860 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:09.861 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-31T02:51:09.861 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-31T02:51:09.861 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x 2024-08-31T02:51:10.168 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:09 smithi102 systemd[1]: Stopping Ceph mgr.x for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:11.668 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:51:11.328+0000 7ffb6fb5f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:11.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x[64795]: 2024-08-31T02:51:11.328+0000 7ffb6fb5f640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-08-31T02:51:11.669 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:11 smithi102 podman[95867]: 2024-08-31 02:51:11.370981407 +0000 UTC m=+0.228880087 container died d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-34b9d27, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-31T02:51:12.008 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:11 smithi102 podman[95867]: 2024-08-31 02:51:11.806471002 +0000 UTC m=+0.664369678 container cleanup d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, RELEASE=squid-34b9d27, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-08-31T02:51:12.008 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:11 smithi102 bash[95867]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x 2024-08-31T02:51:12.284 INFO:journalctl@ceph.mgr.x.smithi102.stdout:Aug 31 02:51:12 smithi102 podman[95898]: 2024-08-31 02:51:12.007997985 +0000 UTC m=+0.638544176 container remove d17af6ad69a7372c06229264564bef996f72e122879767411bef9a8fb1e9b0fc (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-31T02:51:12.835 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@mgr.x.service' 2024-08-31T02:51:12.875 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:12.876 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-31T02:51:12.876 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-31T02:51:12.876 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0 2024-08-31T02:51:13.172 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:12 smithi070 systemd[1]: Stopping Ceph osd.0 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:13.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:51:13.171+0000 7f4be4f04640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:13.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:51:13.171+0000 7f4be4f04640 -1 osd.0 125 *** Got signal Terminated *** 2024-08-31T02:51:13.449 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:13 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0[119982]: 2024-08-31T02:51:13.171+0000 7f4be4f04640 -1 osd.0 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:14.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:14.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:14.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:14.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:19.024 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:18 smithi070 podman[125480]: 2024-08-31 02:51:18.726360095 +0000 UTC m=+5.654100049 container died 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0) 2024-08-31T02:51:19.415 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:19 smithi070 podman[125480]: 2024-08-31 02:51:19.205097081 +0000 UTC m=+6.132837024 container cleanup 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=squid-34b9d27, GIT_BRANCH=HEAD) 2024-08-31T02:51:19.416 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:19 smithi070 bash[125480]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0 2024-08-31T02:51:19.699 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:19 smithi070 podman[125961]: 2024-08-31 02:51:19.41476078 +0000 UTC m=+0.686727291 container remove 23c731c518c0cbfc0b7b567703f174124ff439840fe21cb5621cba8a082a4604 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-34b9d27, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-08-31T02:51:20.134 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:19.920355649 +0000 UTC m=+0.195492933 image pull 72d622302cc2dfd9a14fce59282763f83eedbe02409f698c76a88e3ae2f36da3 quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769 2024-08-31T02:51:20.822 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:20.477902642 +0000 UTC m=+0.753039914 container create 79238692dd2d9e1a13b62bc1c510d23ca359656db0b2b227eecebd1693114457 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-31T02:51:20.823 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:20.722191089 +0000 UTC m=+0.997328361 container init 79238692dd2d9e1a13b62bc1c510d23ca359656db0b2b227eecebd1693114457 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-34b9d27, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-08-31T02:51:20.823 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:20.725537516 +0000 UTC m=+1.000674797 container start 79238692dd2d9e1a13b62bc1c510d23ca359656db0b2b227eecebd1693114457 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=squid-34b9d27, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:51:20.823 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:20.822269562 +0000 UTC m=+1.097406854 container attach 79238692dd2d9e1a13b62bc1c510d23ca359656db0b2b227eecebd1693114457 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-34b9d27) 2024-08-31T02:51:21.199 INFO:journalctl@ceph.osd.0.smithi070.stdout:Aug 31 02:51:20 smithi070 podman[126061]: 2024-08-31 02:51:20.931352575 +0000 UTC m=+1.206489856 container died 79238692dd2d9e1a13b62bc1c510d23ca359656db0b2b227eecebd1693114457 (image=quay.ceph.io/ceph-ci/ceph@sha256:8b075137a5ab0999d974fdcffab872d229d50e3354e13aec412ebf6380176769, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-34b9d27, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-31T02:51:22.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.0.service' 2024-08-31T02:51:22.132 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:22.132 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-31T02:51:22.132 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-31T02:51:22.132 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.1 2024-08-31T02:51:22.450 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:22 smithi070 systemd[1]: Stopping Ceph osd.1 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:22.450 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1[23250]: 2024-08-31T02:51:22.441+0000 7fbcafb5a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:22.450 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1[23250]: 2024-08-31T02:51:22.441+0000 7fbcafb5a700 -1 osd.1 125 *** Got signal Terminated *** 2024-08-31T02:51:22.450 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:22 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1[23250]: 2024-08-31T02:51:22.441+0000 7fbcafb5a700 -1 osd.1 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:24.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:24.243Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:24.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:24.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:24.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:24.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:28.080 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:27 smithi070 podman[126183]: 2024-08-31 02:51:27.817154267 +0000 UTC m=+5.591720093 container died 2ad147a0039cc19e47e9144fdc490e272121743bb47156f09301bd369098778d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, ceph=True, RELEASE=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870) 2024-08-31T02:51:28.449 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126183]: 2024-08-31 02:51:28.262801695 +0000 UTC m=+6.037367511 container cleanup 2ad147a0039cc19e47e9144fdc490e272121743bb47156f09301bd369098778d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container) 2024-08-31T02:51:28.449 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 bash[126183]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1 2024-08-31T02:51:28.450 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 bash[126281]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.1" found: no such container 2024-08-31T02:51:28.814 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126265]: 2024-08-31 02:51:28.530932249 +0000 UTC m=+0.712485576 container remove 2ad147a0039cc19e47e9144fdc490e272121743bb47156f09301bd369098778d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8, release=754, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., ceph=True) 2024-08-31T02:51:28.814 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126340]: 2024-08-31 02:51:28.598413349 +0000 UTC m=+0.017622327 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:51:28.815 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126340]: 2024-08-31 02:51:28.714978456 +0000 UTC m=+0.134187421 container create 6b8260907abecffde5eaaa83e23abe0db2abef38d2207e8babde0956e75dfd8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_CLEAN=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, version=8, name=centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-08-31T02:51:29.200 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126340]: 2024-08-31 02:51:28.925917487 +0000 UTC m=+0.345126453 container init 6b8260907abecffde5eaaa83e23abe0db2abef38d2207e8babde0956e75dfd8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1-deactivate, architecture=x86_64, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-08-31T02:51:29.200 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:28 smithi070 podman[126340]: 2024-08-31 02:51:28.929209149 +0000 UTC m=+0.348418131 container start 6b8260907abecffde5eaaa83e23abe0db2abef38d2207e8babde0956e75dfd8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , name=centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True) 2024-08-31T02:51:29.200 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:29 smithi070 podman[126340]: 2024-08-31 02:51:29.000716094 +0000 UTC m=+0.419925067 container attach 6b8260907abecffde5eaaa83e23abe0db2abef38d2207e8babde0956e75dfd8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git) 2024-08-31T02:51:29.512 INFO:journalctl@ceph.osd.1.smithi070.stdout:Aug 31 02:51:29 smithi070 podman[126340]: 2024-08-31 02:51:29.206445335 +0000 UTC m=+0.625654315 container died 6b8260907abecffde5eaaa83e23abe0db2abef38d2207e8babde0956e75dfd8f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-1-deactivate, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc.) 2024-08-31T02:51:29.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.1.service' 2024-08-31T02:51:29.946 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:29.947 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-31T02:51:29.947 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-31T02:51:29.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.2 2024-08-31T02:51:30.158 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:30 smithi070 systemd[1]: Stopping Ceph osd.2 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:30.449 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2[26174]: 2024-08-31T02:51:30.156+0000 7f8134dbc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:30.449 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2[26174]: 2024-08-31T02:51:30.156+0000 7f8134dbc700 -1 osd.2 125 *** Got signal Terminated *** 2024-08-31T02:51:30.450 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:30 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2[26174]: 2024-08-31T02:51:30.156+0000 7f8134dbc700 -1 osd.2 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:34.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:34.243Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:34.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:34.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:34.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:34 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:34.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:35.949 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:35 smithi070 podman[126419]: 2024-08-31 02:51:35.556879806 +0000 UTC m=+5.516718342 container died f13f7407a65c811c10040bc2e3ccf4f1d1167dac7f3a48a4268b5e026f65c231 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2, maintainer=Guillaume Abrioux , vcs-type=git, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, release=754, name=centos-stream, io.openshift.expose-services=, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True) 2024-08-31T02:51:36.293 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:35 smithi070 podman[126419]: 2024-08-31 02:51:35.973046249 +0000 UTC m=+5.932884826 container cleanup f13f7407a65c811c10040bc2e3ccf4f1d1167dac7f3a48a4268b5e026f65c231 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, RELEASE=HEAD) 2024-08-31T02:51:36.293 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:35 smithi070 bash[126419]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2 2024-08-31T02:51:36.293 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126436]: 2024-08-31 02:51:36.174321956 +0000 UTC m=+0.619110255 container remove f13f7407a65c811c10040bc2e3ccf4f1d1167dac7f3a48a4268b5e026f65c231 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, name=centos-stream) 2024-08-31T02:51:36.294 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 bash[126448]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.2" found: no such container 2024-08-31T02:51:36.557 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.323365231 +0000 UTC m=+0.017456189 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:51:36.557 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.42499552 +0000 UTC m=+0.119086458 container create a551e771f5299e7297ea6b539a7e9365197dd3108deffe415c23b5e4c6333d18 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2-deactivate, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:51:36.949 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.652795803 +0000 UTC m=+0.346886745 container init a551e771f5299e7297ea6b539a7e9365197dd3108deffe415c23b5e4c6333d18 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, vendor=Red Hat, Inc., vcs-type=git, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, ceph=True, release=754) 2024-08-31T02:51:36.950 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.656500274 +0000 UTC m=+0.350591203 container start a551e771f5299e7297ea6b539a7e9365197dd3108deffe415c23b5e4c6333d18 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2-deactivate, io.openshift.expose-services=, io.buildah.version=1.19.8, distribution-scope=public, architecture=x86_64, vendor=Red Hat, Inc., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-08-31T02:51:36.950 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.74447972 +0000 UTC m=+0.438570656 container attach a551e771f5299e7297ea6b539a7e9365197dd3108deffe415c23b5e4c6333d18 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2-deactivate, io.openshift.tags=base centos centos-stream, release=754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public) 2024-08-31T02:51:37.272 INFO:journalctl@ceph.osd.2.smithi070.stdout:Aug 31 02:51:36 smithi070 podman[126552]: 2024-08-31 02:51:36.961423054 +0000 UTC m=+0.655513991 container died a551e771f5299e7297ea6b539a7e9365197dd3108deffe415c23b5e4c6333d18 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-2-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_CLEAN=True, version=8, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-08-31T02:51:37.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.2.service' 2024-08-31T02:51:37.702 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:37.702 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-31T02:51:37.702 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-31T02:51:37.702 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.3 2024-08-31T02:51:37.918 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:37 smithi070 systemd[1]: Stopping Ceph osd.3 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:38.199 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:37.917+0000 7f63e3c3d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:38.200 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:37.917+0000 7f63e3c3d700 -1 osd.3 125 *** Got signal Terminated *** 2024-08-31T02:51:38.200 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:37 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:37.917+0000 7f63e3c3d700 -1 osd.3 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:40.256 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:39 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:39.942+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:40.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:40.255+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:41.199 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:40 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:40.920+0000 7f63de5d4700 -1 osd.3 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.934805+0000 front 2024-08-31T02:51:17.934947+0000 (oldest deadline 2024-08-31T02:51:40.135484+0000) 2024-08-31T02:51:41.267 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:40.936+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:41.267 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:41.153+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:41.658 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:41.266+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:42.199 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:41 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:41.935+0000 7f63de5d4700 -1 osd.3 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.934805+0000 front 2024-08-31T02:51:17.934947+0000 (oldest deadline 2024-08-31T02:51:40.135484+0000) 2024-08-31T02:51:42.219 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:42.202+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:42.220 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:41 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:41.955+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:42.220 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:42.147+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:42.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:42.309+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:43.179 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:42 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3[29155]: 2024-08-31T02:51:42.895+0000 7f63de5d4700 -1 osd.3 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.934805+0000 front 2024-08-31T02:51:17.934947+0000 (oldest deadline 2024-08-31T02:51:40.135484+0000) 2024-08-31T02:51:43.352 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:43.158+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:43.353 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:43.118+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:43.353 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:42 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:42.929+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:43.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:43.351+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:44.092 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:43 smithi070 podman[126783]: 2024-08-31 02:51:43.8265726 +0000 UTC m=+6.038447292 container died d13f8745f80ec219f182345b78ac19420946687ec2165577955af540e3f58aa0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, name=centos-stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-08-31T02:51:44.343 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:44.105+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:44.344 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:44.163+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:44.344 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:43 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:43.951+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:44.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:44.244Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:44.449 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:44.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:44.450 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:44 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:44.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:44.450 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[126783]: 2024-08-31 02:51:44.182954759 +0000 UTC m=+6.394829450 container cleanup d13f8745f80ec219f182345b78ac19420946687ec2165577955af540e3f58aa0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3, RELEASE=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., distribution-scope=public, maintainer=Guillaume Abrioux ) 2024-08-31T02:51:44.450 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 bash[126783]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3 2024-08-31T02:51:44.450 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[127349]: 2024-08-31 02:51:44.334560421 +0000 UTC m=+1.037899570 container remove d13f8745f80ec219f182345b78ac19420946687ec2165577955af540e3f58aa0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, release=754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, ceph=True) 2024-08-31T02:51:44.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:44.342+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:44.765 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 bash[127388]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.3" found: no such container 2024-08-31T02:51:45.055 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[127451]: 2024-08-31 02:51:44.667872656 +0000 UTC m=+0.017323973 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:51:45.055 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[127451]: 2024-08-31 02:51:44.791752251 +0000 UTC m=+0.141203558 container create 476f860773f35e694c8eab70afad002fb538b90eeda03c57377d7f18589edc28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3-deactivate, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.component=centos-stream-container, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., release=754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_BRANCH=HEAD, distribution-scope=public) 2024-08-31T02:51:45.056 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[127451]: 2024-08-31 02:51:44.980523627 +0000 UTC m=+0.329974944 container init 476f860773f35e694c8eab70afad002fb538b90eeda03c57377d7f18589edc28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-08-31T02:51:45.056 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:44 smithi070 podman[127451]: 2024-08-31 02:51:44.983989259 +0000 UTC m=+0.333440571 container start 476f860773f35e694c8eab70afad002fb538b90eeda03c57377d7f18589edc28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, release=754, io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_CLEAN=True) 2024-08-31T02:51:45.056 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:45 smithi070 podman[127451]: 2024-08-31 02:51:45.05496537 +0000 UTC m=+0.404416675 container attach 476f860773f35e694c8eab70afad002fb538b90eeda03c57377d7f18589edc28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3-deactivate, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, name=centos-stream, vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8) 2024-08-31T02:51:45.373 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:45.162+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:45.374 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:45.103+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:45.374 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:44 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:44.954+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:45.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:45.372+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:45.699 INFO:journalctl@ceph.osd.3.smithi070.stdout:Aug 31 02:51:45 smithi070 podman[127451]: 2024-08-31 02:51:45.375328301 +0000 UTC m=+0.724779605 container died 476f860773f35e694c8eab70afad002fb538b90eeda03c57377d7f18589edc28 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-type=git, name=centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, ceph=True, release=754, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-08-31T02:51:46.134 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.3.service' 2024-08-31T02:51:46.174 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:46.175 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-31T02:51:46.175 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-31T02:51:46.175 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.4 2024-08-31T02:51:46.229 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:46.123+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:46.229 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:46.199+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:46.229 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:45 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:45.969+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:46.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:46.330+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:46.668 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:46 smithi102 systemd[1]: Stopping Ceph osd.4 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:46.668 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:46.335+0000 7f583e57f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:46.669 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:46.335+0000 7f583e57f700 -1 osd.4 125 *** Got signal Terminated *** 2024-08-31T02:51:46.669 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:46.335+0000 7f583e57f700 -1 osd.4 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:47.360 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:47.146+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:47.361 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:47.160+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:47.361 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:46 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:46.927+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:47.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:47.359+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:48.408 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:48.138+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:48.408 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:48.161+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:48.409 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:47 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:47.977+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:48.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:48.407+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:49.150 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:48.930+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:49.150 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:48 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:48.930+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:49.418 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:49.211+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:49.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:49.149+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:49.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:49.380+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:49.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:49.380+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:50.109 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:49.966+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:50.109 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:49 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:49.966+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:50.379 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:50.143+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:50.380 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:50.192+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:50.380 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:50.192+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.138116+0000 front 2024-08-31T02:51:25.138132+0000 (oldest deadline 2024-08-31T02:51:49.238302+0000) 2024-08-31T02:51:50.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:50.378+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:50.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:50.378+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:51.168 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:51.110+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:51.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:50.925+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:51.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:50 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:50.925+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:51.580 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:51.189+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.736282+0000 front 2024-08-31T02:51:17.736267+0000 (oldest deadline 2024-08-31T02:51:41.236165+0000) 2024-08-31T02:51:51.581 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4[17839]: 2024-08-31T02:51:51.189+0000 7f583b71b700 -1 osd.4 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.138116+0000 front 2024-08-31T02:51:25.138132+0000 (oldest deadline 2024-08-31T02:51:49.238302+0000) 2024-08-31T02:51:51.581 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:51.382+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:51.581 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:51.382+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:51.918 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:51 smithi102 podman[97056]: 2024-08-31 02:51:51.779807561 +0000 UTC m=+5.540917579 container died 430432c2542906c73ee9bac6c2e8891c0607d2bf5a389bb458bc085853ed4717 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD) 2024-08-31T02:51:52.249 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:52.139+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:52.250 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:52.139+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:52.250 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:51.920+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:52.250 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:51 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:51.920+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:52.543 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:52.379+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:52.543 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:52.379+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:52.543 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97056]: 2024-08-31 02:51:52.249737601 +0000 UTC m=+6.010847618 container cleanup 430432c2542906c73ee9bac6c2e8891c0607d2bf5a389bb458bc085853ed4717 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, distribution-scope=public, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, version=8, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-08-31T02:51:52.544 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 bash[97056]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4 2024-08-31T02:51:52.544 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 bash[97200]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.4" found: no such container 2024-08-31T02:51:52.862 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97183]: 2024-08-31 02:51:52.542945262 +0000 UTC m=+0.762147101 container remove 430432c2542906c73ee9bac6c2e8891c0607d2bf5a389bb458bc085853ed4717 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4, GIT_BRANCH=HEAD, RELEASE=HEAD, vcs-type=git, name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:51:52.862 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97258]: 2024-08-31 02:51:52.633422939 +0000 UTC m=+0.017094698 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:51:52.862 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97258]: 2024-08-31 02:51:52.750299303 +0000 UTC m=+0.133971060 container create c2860be0642df9aa30203982f13836e33a01b3bd42c9bc31bd0529fd5ee5b365 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4-deactivate, ceph=True, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, name=centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:51:53.141 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:52.955+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:53.141 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:52 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:52.955+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:53.142 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97258]: 2024-08-31 02:51:52.962988878 +0000 UTC m=+0.346660638 container init c2860be0642df9aa30203982f13836e33a01b3bd42c9bc31bd0529fd5ee5b365 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4-deactivate, com.redhat.component=centos-stream-container, release=754, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-08-31T02:51:53.142 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:52 smithi102 podman[97258]: 2024-08-31 02:51:52.966283505 +0000 UTC m=+0.349955266 container start c2860be0642df9aa30203982f13836e33a01b3bd42c9bc31bd0529fd5ee5b365 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4-deactivate, version=8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8) 2024-08-31T02:51:53.142 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:53 smithi102 podman[97258]: 2024-08-31 02:51:53.046280795 +0000 UTC m=+0.429952563 container attach c2860be0642df9aa30203982f13836e33a01b3bd42c9bc31bd0529fd5ee5b365 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4-deactivate, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870) 2024-08-31T02:51:53.409 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:53.140+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:53.409 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:53.140+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:53.410 INFO:journalctl@ceph.osd.4.smithi102.stdout:Aug 31 02:51:53 smithi102 podman[97258]: 2024-08-31 02:51:53.261826455 +0000 UTC m=+0.645498221 container died c2860be0642df9aa30203982f13836e33a01b3bd42c9bc31bd0529fd5ee5b365 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-type=git, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, io.buildah.version=1.19.8) 2024-08-31T02:51:53.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:53.408+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:53.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:53.408+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:53.875 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.4.service' 2024-08-31T02:51:53.917 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:51:53.917 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-31T02:51:53.917 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-31T02:51:53.918 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.5 2024-08-31T02:51:54.095 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:53.972+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:54.096 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:53 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:53.972+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:54.096 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 systemd[1]: Stopping Ceph osd.5 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:51:54.096 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:54.094+0000 7f23a5c0e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:51:54.096 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:54.094+0000 7f23a5c0e700 -1 osd.5 125 *** Got signal Terminated *** 2024-08-31T02:51:54.096 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:54.094+0000 7f23a5c0e700 -1 osd.5 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:51:54.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:54.111+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:54.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:54.111+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:54.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:54.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:54.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:54.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:51:54.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:51:54 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:51:54.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:51:54.918 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:54.426+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:54.918 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:54.426+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:55.260 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:55.149+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:55.260 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:55.149+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:55.261 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:54.957+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:55.261 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:54 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:54.957+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:55.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:55.376+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:55.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:55.376+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:56.289 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:56.124+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:56.290 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:56.124+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:56.290 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:55.983+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:56.290 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:55 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:55.983+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:56.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:56.337+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:56.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:56.337+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:57.083 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:56.943+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:57.083 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:56 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:56.943+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:57.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:57.082+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:57.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:57.082+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:57.418 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:57.082+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.797933+0000 front 2024-08-31T02:51:34.797937+0000 (oldest deadline 2024-08-31T02:51:56.497886+0000) 2024-08-31T02:51:57.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:57.318+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:57.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:57.318+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:57.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:57.318+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:51:58.311 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:57.957+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:58.312 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:57 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:57.957+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:58.312 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:58.119+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:58.312 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:58.119+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:58.312 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:58.119+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.797933+0000 front 2024-08-31T02:51:34.797937+0000 (oldest deadline 2024-08-31T02:51:56.497886+0000) 2024-08-31T02:51:58.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:58.311+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:58.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:58.311+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:58.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:58.311+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:51:59.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:58.915+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:51:59.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:58 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:58.915+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:51:59.168 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:59.080+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.895647+0000 front 2024-08-31T02:51:17.895771+0000 (oldest deadline 2024-08-31T02:51:40.795529+0000) 2024-08-31T02:51:59.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:59.080+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.997165+0000 front 2024-08-31T02:51:25.997076+0000 (oldest deadline 2024-08-31T02:51:51.896923+0000) 2024-08-31T02:51:59.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5[20692]: 2024-08-31T02:51:59.080+0000 7f23a1da8700 -1 osd.5 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.797933+0000 front 2024-08-31T02:51:34.797937+0000 (oldest deadline 2024-08-31T02:51:56.497886+0000) 2024-08-31T02:51:59.511 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:59.329+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:51:59.511 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:59.329+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:51:59.511 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:51:59.329+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:51:59.800 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 podman[97454]: 2024-08-31 02:51:59.510696064 +0000 UTC m=+5.501351770 container died a2668180d986b9988afa5de891a80f52ae2ff42cf115d9c9b1976d240a883143 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, RELEASE=HEAD, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , architecture=x86_64, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, release=754) 2024-08-31T02:52:00.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:59.902+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:00.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:51:59 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:51:59.902+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:00.168 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 podman[97454]: 2024-08-31 02:51:59.999353842 +0000 UTC m=+5.990009547 container cleanup a2668180d986b9988afa5de891a80f52ae2ff42cf115d9c9b1976d240a883143 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5, GIT_CLEAN=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, ceph=True, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-08-31T02:52:00.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:51:59 smithi102 bash[97454]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5 2024-08-31T02:52:00.565 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[97976]: 2024-08-31 02:52:00.284604476 +0000 UTC m=+0.776644504 container remove a2668180d986b9988afa5de891a80f52ae2ff42cf115d9c9b1976d240a883143 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, distribution-scope=public) 2024-08-31T02:52:00.565 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 bash[98002]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.5" found: no such container 2024-08-31T02:52:00.566 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:00.303+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:00.566 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:00.303+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:00.566 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:00.303+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:00.880 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[98058]: 2024-08-31 02:52:00.468485626 +0000 UTC m=+0.016528923 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:52:00.880 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[98058]: 2024-08-31 02:52:00.635873753 +0000 UTC m=+0.183917035 container create e340ab633105d43f7cf028119903cecfb4e68fb5872bf4fb490a92f1ef0d965c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5-deactivate, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-08-31T02:52:01.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:00.897+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:01.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:00 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:00.897+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:01.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[98058]: 2024-08-31 02:52:00.880399368 +0000 UTC m=+0.428442669 container init e340ab633105d43f7cf028119903cecfb4e68fb5872bf4fb490a92f1ef0d965c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5-deactivate, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, architecture=x86_64, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vcs-type=git, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8, io.openshift.expose-services=, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-08-31T02:52:01.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[98058]: 2024-08-31 02:52:00.884423791 +0000 UTC m=+0.432467074 container start e340ab633105d43f7cf028119903cecfb4e68fb5872bf4fb490a92f1ef0d965c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-08-31T02:52:01.169 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:00 smithi102 podman[98058]: 2024-08-31 02:52:00.988332848 +0000 UTC m=+0.536376144 container attach e340ab633105d43f7cf028119903cecfb4e68fb5872bf4fb490a92f1ef0d965c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, architecture=x86_64, GIT_CLEAN=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8) 2024-08-31T02:52:01.467 INFO:journalctl@ceph.osd.5.smithi102.stdout:Aug 31 02:52:01 smithi102 podman[98058]: 2024-08-31 02:52:01.170971914 +0000 UTC m=+0.719015240 container died e340ab633105d43f7cf028119903cecfb4e68fb5872bf4fb490a92f1ef0d965c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-5-deactivate, io.k8s.display-name=CentOS Stream 8, ceph=True, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-08-31T02:52:01.467 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:01.345+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:01.467 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:01.345+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:01.467 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:01.345+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:01.857 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.5.service' 2024-08-31T02:52:01.901 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:52:01.902 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-31T02:52:01.902 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-31T02:52:01.902 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.6 2024-08-31T02:52:02.137 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:01.904+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:01.904+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:01 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:01.904+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:01 smithi102 systemd[1]: Stopping Ceph osd.6 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.136+0000 7fcf31c29700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.136+0000 7fcf31c29700 -1 osd.6 125 *** Got signal Terminated *** 2024-08-31T02:52:02.138 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.136+0000 7fcf31c29700 -1 osd.6 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:52:02.418 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:02.316+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:02.418 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:02.316+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:02.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:02.316+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:03.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.873+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:03.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.873+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:03.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:02 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:02.873+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:03.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:03.334+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:03.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:03.334+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:03.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:03.334+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:04.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:03.885+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:04.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:03.885+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:04.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:03 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:03.885+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:04.450 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:04.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:52:04.450 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:04.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:52:04.450 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:04 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:04.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:52:04.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:04.360+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:04.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:04.360+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:04.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:04.360+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:04.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:04.360+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:05.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:04.840+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:05.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:04.840+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:05.169 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:04.840+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:05.169 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:04 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:04.840+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.232368+0000 front 2024-08-31T02:51:41.232300+0000 (oldest deadline 2024-08-31T02:52:04.132272+0000) 2024-08-31T02:52:05.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:05.324+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:05.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:05.324+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:05.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:05.324+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:05.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:05.324+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:06.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:05.846+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:06.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:05.846+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:06.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:05.846+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:06.169 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:05 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:05.846+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.232368+0000 front 2024-08-31T02:51:41.232300+0000 (oldest deadline 2024-08-31T02:52:04.132272+0000) 2024-08-31T02:52:06.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:06.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:06.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:06.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:06.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:06.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:06.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:06.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:07.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:06.804+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:17.829337+0000 front 2024-08-31T02:51:17.829601+0000 (oldest deadline 2024-08-31T02:51:38.929507+0000) 2024-08-31T02:52:07.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:06.804+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:23.030606+0000 front 2024-08-31T02:51:23.030510+0000 (oldest deadline 2024-08-31T02:51:48.330412+0000) 2024-08-31T02:52:07.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:06.804+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:31.831195+0000 front 2024-08-31T02:51:31.831142+0000 (oldest deadline 2024-08-31T02:52:01.231666+0000) 2024-08-31T02:52:07.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:06 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6[23586]: 2024-08-31T02:52:06.804+0000 7fcf2f5c6700 -1 osd.6 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.232368+0000 front 2024-08-31T02:51:41.232300+0000 (oldest deadline 2024-08-31T02:52:04.132272+0000) 2024-08-31T02:52:07.573 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:07.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:07.573 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:07.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:07.573 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:07.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:07.573 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:07 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:07.312+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:07.849 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:07 smithi102 podman[98137]: 2024-08-31 02:52:07.572257766 +0000 UTC m=+5.568088656 container died d4eb46bd6282a32592d5a731f09b630297a96a0d3aa901962df2b24893485d94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-08-31T02:52:08.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:07 smithi102 podman[98137]: 2024-08-31 02:52:07.946768007 +0000 UTC m=+5.942598887 container cleanup d4eb46bd6282a32592d5a731f09b630297a96a0d3aa901962df2b24893485d94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_CLEAN=True, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=) 2024-08-31T02:52:08.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:07 smithi102 bash[98137]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6 2024-08-31T02:52:08.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:07 smithi102 bash[98167]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.6" found: no such container 2024-08-31T02:52:08.496 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:08.285+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:08.496 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:08.285+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:08.496 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:08.285+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:08.496 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:08 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:08.285+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:08.497 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98152]: 2024-08-31 02:52:08.197927966 +0000 UTC m=+0.618725711 container remove d4eb46bd6282a32592d5a731f09b630297a96a0d3aa901962df2b24893485d94 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-08-31T02:52:08.497 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.278814809 +0000 UTC m=+0.017275426 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:52:08.497 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.382012498 +0000 UTC m=+0.120473154 container create 88cd26067f46d5fa8116b8df73fc2baf7b4183ce3b181245d1f23b838d01eca4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6-deactivate, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., name=centos-stream) 2024-08-31T02:52:08.879 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.593111365 +0000 UTC m=+0.331571991 container init 88cd26067f46d5fa8116b8df73fc2baf7b4183ce3b181245d1f23b838d01eca4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754, distribution-scope=public, io.openshift.expose-services=, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True) 2024-08-31T02:52:08.880 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.596831485 +0000 UTC m=+0.335292099 container start 88cd26067f46d5fa8116b8df73fc2baf7b4183ce3b181245d1f23b838d01eca4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6-deactivate, ceph=True, architecture=x86_64, RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, release=754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-08-31T02:52:08.880 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.659152732 +0000 UTC m=+0.397613356 container attach 88cd26067f46d5fa8116b8df73fc2baf7b4183ce3b181245d1f23b838d01eca4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6-deactivate, vendor=Red Hat, Inc., ceph=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream) 2024-08-31T02:52:09.168 INFO:journalctl@ceph.osd.6.smithi102.stdout:Aug 31 02:52:08 smithi102 podman[98228]: 2024-08-31 02:52:08.879497543 +0000 UTC m=+0.617958168 container died 88cd26067f46d5fa8116b8df73fc2baf7b4183ce3b181245d1f23b838d01eca4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-6-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, version=8, GIT_BRANCH=HEAD, distribution-scope=public, release=754, name=centos-stream) 2024-08-31T02:52:09.479 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.261+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:09.480 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.261+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:09.480 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.261+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:09.480 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.261+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:09.480 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:09.434Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:52:09.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:09.434Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:52:09.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:09.436Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:52:09.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:09.437Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:52:09.481 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:09.437Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.102:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.102:8765: connect: connection refused" 2024-08-31T02:52:09.499 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.6.service' 2024-08-31T02:52:09.539 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:52:09.540 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-31T02:52:09.540 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-31T02:52:09.540 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.7 2024-08-31T02:52:09.838 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 systemd[1]: Stopping Ceph osd.7 for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:10.099 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.933+0000 7f0b88ad1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-31T02:52:10.099 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.933+0000 7f0b88ad1700 -1 osd.7 125 *** Got signal Terminated *** 2024-08-31T02:52:10.100 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:09 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:09.933+0000 7f0b88ad1700 -1 osd.7 125 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T02:52:10.361 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:10.281+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:10.361 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:10.281+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:10.361 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:10.281+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:10.362 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:10 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:10.281+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:11.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:11.301+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:11.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:11.301+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:11.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:11.301+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:11.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:11 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:11.301+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:12.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:12.349+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:12.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:12.349+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:12.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:12.349+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:12.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:12 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:12.349+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:13.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:13.326+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:13.761 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:13.326+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:13.761 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:13.326+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:13.761 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:13 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:13.326+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:14.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:14.357+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6804 osd.0 since back 2024-08-31T02:51:13.181162+0000 front 2024-08-31T02:51:13.181171+0000 (oldest deadline 2024-08-31T02:51:39.581672+0000) 2024-08-31T02:52:14.668 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:14.357+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6814 osd.1 since back 2024-08-31T02:51:25.482587+0000 front 2024-08-31T02:51:25.482657+0000 (oldest deadline 2024-08-31T02:51:48.982396+0000) 2024-08-31T02:52:14.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:14.357+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6822 osd.2 since back 2024-08-31T02:51:34.183624+0000 front 2024-08-31T02:51:34.183582+0000 (oldest deadline 2024-08-31T02:51:56.483507+0000) 2024-08-31T02:52:14.669 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:14 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7[26525]: 2024-08-31T02:52:14.357+0000 7f0b82466700 -1 osd.7 125 heartbeat_check: no reply from 172.21.15.70:6830 osd.3 since back 2024-08-31T02:51:41.184347+0000 front 2024-08-31T02:51:41.184358+0000 (oldest deadline 2024-08-31T02:52:04.084373+0000) 2024-08-31T02:52:14.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:14.247Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:52:14.699 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:14.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:52:14.700 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:14 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:14.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:52:16.267 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:16 smithi102 podman[98322]: 2024-08-31 02:52:16.000860491 +0000 UTC m=+6.369437989 container died 72b7b2af68b0bc0a8c2e3d0c698b88ca8096c07382312fe7a5946f0e8d1cb1b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, vendor=Red Hat, Inc., GIT_CLEAN=True, version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True) 2024-08-31T02:52:16.636 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:16 smithi102 podman[98322]: 2024-08-31 02:52:16.266980484 +0000 UTC m=+6.635557979 container cleanup 72b7b2af68b0bc0a8c2e3d0c698b88ca8096c07382312fe7a5946f0e8d1cb1b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:52:16.636 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:16 smithi102 bash[98322]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7 2024-08-31T02:52:16.637 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:16 smithi102 podman[98900]: 2024-08-31 02:52:16.409920299 +0000 UTC m=+0.937927996 container remove 72b7b2af68b0bc0a8c2e3d0c698b88ca8096c07382312fe7a5946f0e8d1cb1b5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, name=centos-stream, release=754, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream) 2024-08-31T02:52:16.637 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:16 smithi102 bash[98966]: Error: no container with name or ID "ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd.7" found: no such container 2024-08-31T02:52:18.026 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:17 smithi102 podman[99138]: 2024-08-31 02:52:17.683440278 +0000 UTC m=+0.019076708 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-08-31T02:52:18.027 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:17 smithi102 podman[99138]: 2024-08-31 02:52:17.791496933 +0000 UTC m=+0.127133355 container create 8bfea60e47ee5f211df37e1808cf878d5d7afdb168d424867b2ce434667e162d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, name=centos-stream) 2024-08-31T02:52:18.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:18 smithi102 podman[99138]: 2024-08-31 02:52:18.135499115 +0000 UTC m=+0.471135549 container init 8bfea60e47ee5f211df37e1808cf878d5d7afdb168d424867b2ce434667e162d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-08-31T02:52:18.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:18 smithi102 podman[99138]: 2024-08-31 02:52:18.138770096 +0000 UTC m=+0.474406517 container start 8bfea60e47ee5f211df37e1808cf878d5d7afdb168d424867b2ce434667e162d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container) 2024-08-31T02:52:18.419 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:18 smithi102 podman[99138]: 2024-08-31 02:52:18.218600375 +0000 UTC m=+0.554236809 container attach 8bfea60e47ee5f211df37e1808cf878d5d7afdb168d424867b2ce434667e162d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7-deactivate, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, RELEASE=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-type=git, release=754, version=8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0) 2024-08-31T02:52:18.764 INFO:journalctl@ceph.osd.7.smithi102.stdout:Aug 31 02:52:18 smithi102 podman[99138]: 2024-08-31 02:52:18.421771708 +0000 UTC m=+0.757408129 container died 8bfea60e47ee5f211df37e1808cf878d5d7afdb168d424867b2ce434667e162d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-osd-7-deactivate, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-type=git) 2024-08-31T02:52:19.048 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@osd.7.service' 2024-08-31T02:52:19.087 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:52:19.087 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-31T02:52:19.087 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-31T02:52:19.088 DEBUG:teuthology.orchestra.run.smithi102:> sudo systemctl stop ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a 2024-08-31T02:52:19.408 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 systemd[1]: Stopping Ceph prometheus.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:19.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:52:19.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-31T02:52:19.668 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.408Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.412Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-31T02:52:19.669 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.412Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-31T02:52:19.670 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a[86008]: ts=2024-08-31T02:52:19.412Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-31T02:52:19.670 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 podman[99255]: 2024-08-31 02:52:19.418701716 +0000 UTC m=+0.113381264 container died 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:20.134 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 podman[99255]: 2024-08-31 02:52:19.883227437 +0000 UTC m=+0.577906972 container cleanup 8c3920246ae0ff8ccec898edc9435cb02515c8f146f1f5fae0aae078c9f08951 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:20.134 INFO:journalctl@ceph.prometheus.a.smithi102.stdout:Aug 31 02:52:19 smithi102 bash[99255]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-prometheus-a 2024-08-31T02:52:20.300 DEBUG:teuthology.orchestra.run.smithi102:> sudo pkill -f 'journalctl -f -n 0 -u ceph-862b0c56-673e-11ef-bcd4-c7b262605968@prometheus.a.service' 2024-08-31T02:52:20.333 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-31T02:52:20.334 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-31T02:52:20.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 --force --keep-logs 2024-08-31T02:52:23.200 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 systemd[1]: Stopping Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:23.517 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 podman[128801]: 2024-08-31 02:52:23.262368772 +0000 UTC m=+0.120606915 container died 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:23.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 podman[128801]: 2024-08-31 02:52:23.700498249 +0000 UTC m=+0.558736433 container cleanup 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:23.949 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 bash[128801]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a 2024-08-31T02:52:24.248 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 podman[128815]: 2024-08-31 02:52:23.984699616 +0000 UTC m=+0.726072401 container remove 95e4443a43cb6844ec4380eeb057b69c3cdb5c09bc0c2515de9ddb7284164959 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:24.249 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:23 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:52:24.249 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:24 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-31T02:52:24.249 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:24 smithi070 systemd[1]: Stopped Ceph node-exporter.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:52:24.249 INFO:journalctl@ceph.node-exporter.a.smithi070.stdout:Aug 31 02:52:24 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.a.service: Consumed 1.637s CPU time. 2024-08-31T02:52:24.529 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:24.248Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:52:24.530 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:24.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi070:8443/api/prometheus_receiver\": dial tcp 172.21.15.70:8443: connect: connection refused" 2024-08-31T02:52:24.530 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:24.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi102.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.102:8443: connect: connection refused" 2024-08-31T02:52:24.530 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 systemd[1]: Stopping Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:24.530 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a[91041]: ts=2024-08-31T02:52:24.529Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-31T02:52:24.949 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:24 smithi070 podman[128923]: 2024-08-31 02:52:24.580388563 +0000 UTC m=+0.150262464 container died df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:25.297 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 podman[128923]: 2024-08-31 02:52:25.104940964 +0000 UTC m=+0.674814857 container cleanup df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:25.298 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 bash[128923]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a 2024-08-31T02:52:25.659 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 podman[128936]: 2024-08-31 02:52:25.297719398 +0000 UTC m=+0.710005853 container remove df358a3d49e78106a92d80cedc5f1dea8b6a03efd45aa0259bb114a9f42c209d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-31T02:52:25.659 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 podman[128936]: 2024-08-31 02:52:25.381511384 +0000 UTC m=+0.793797793 volume remove 518517785fda942ba49bef618cd182446c950087c08cacdb45c276af4054bfcf 2024-08-31T02:52:25.944 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-31T02:52:25.944 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 systemd[1]: Stopped Ceph alertmanager.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:52:25.944 INFO:journalctl@ceph.alertmanager.a.smithi070.stdout:Aug 31 02:52:25 smithi070 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@alertmanager.a.service: Consumed 1.316s CPU time. 2024-08-31T02:52:37.212 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 --force --keep-logs 2024-08-31T02:52:39.668 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:39 smithi102 systemd[1]: Stopping Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:39.669 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:39 smithi102 podman[100401]: 2024-08-31 02:52:39.478348536 +0000 UTC m=+0.100862714 container died f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:52:40.094 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:39 smithi102 podman[100401]: 2024-08-31 02:52:39.876246486 +0000 UTC m=+0.498760703 container cleanup f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:52:40.094 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:39 smithi102 bash[100401]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b 2024-08-31T02:52:40.402 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:40 smithi102 podman[100413]: 2024-08-31 02:52:40.094001343 +0000 UTC m=+0.619133596 container remove f63aad055833a0cd167129bc8ebecb99bb9f1a3df74fa9eea51515aabba5cb18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-31T02:52:40.402 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:40 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-31T02:52:40.402 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:40 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-31T02:52:40.402 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:40 smithi102 systemd[1]: Stopped Ceph node-exporter.b for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:52:40.402 INFO:journalctl@ceph.node-exporter.b.smithi102.stdout:Aug 31 02:52:40 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@node-exporter.b.service: Consumed 1.601s CPU time. 2024-08-31T02:52:40.657 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:40 smithi102 systemd[1]: Stopping Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968... 2024-08-31T02:52:40.658 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=server t=2024-08-31T02:52:40.646842549Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-08-31T02:52:40.658 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=tracing t=2024-08-31T02:52:40.647218162Z level=info msg="Closing tracing" 2024-08-31T02:52:40.658 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:40 smithi102 ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a[76747]: logger=ticker t=2024-08-31T02:52:40.64724163Z level=info msg=stopped last_tick=2024-08-31T02:52:40Z 2024-08-31T02:52:40.918 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:40 smithi102 podman[100521]: 2024-08-31 02:52:40.657379391 +0000 UTC m=+0.116565748 container died 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:52:41.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 podman[100521]: 2024-08-31 02:52:41.230016157 +0000 UTC m=+0.689202511 container cleanup 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:52:41.418 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 bash[100521]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a 2024-08-31T02:52:41.823 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 podman[100537]: 2024-08-31 02:52:41.481343519 +0000 UTC m=+0.821743238 container remove 841fb84c461e9adc282449276222dad299ecb1a215c87a8af342a98c6d20e548 (image=quay.io/ceph/grafana:9.4.12, name=ceph-862b0c56-673e-11ef-bcd4-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-31T02:52:41.824 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-31T02:52:41.824 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 systemd[1]: Stopped Ceph grafana.a for 862b0c56-673e-11ef-bcd4-c7b262605968. 2024-08-31T02:52:41.824 INFO:journalctl@ceph.grafana.a.smithi102.stdout:Aug 31 02:52:41 smithi102 systemd[1]: ceph-862b0c56-673e-11ef-bcd4-c7b262605968@grafana.a.service: Consumed 7.502s CPU time. 2024-08-31T02:52:52.996 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:52:53.031 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-31T02:52:53.060 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-31T02:52:53.061 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi070/crash 2024-08-31T02:52:53.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash -- . 2024-08-31T02:52:53.115 INFO:teuthology.orchestra.run.smithi070.stderr:tar: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash: Cannot open: No such file or directory 2024-08-31T02:52:53.115 INFO:teuthology.orchestra.run.smithi070.stderr:tar: Error is not recoverable: exiting now 2024-08-31T02:52:53.117 DEBUG:teuthology.misc:Transferring archived files from smithi102:/var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi102/crash 2024-08-31T02:52:53.118 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash -- . 2024-08-31T02:52:53.165 INFO:teuthology.orchestra.run.smithi102.stderr:tar: /var/lib/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/crash: Cannot open: No such file or directory 2024-08-31T02:52:53.165 INFO:teuthology.orchestra.run.smithi102.stderr:tar: Error is not recoverable: exiting now 2024-08-31T02:52:53.166 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-31T02:52:53.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-08-31T02:52:53.198 INFO:tasks.cephadm:Compressing logs... 2024-08-31T02:52:53.198 DEBUG:teuthology.orchestra.run.smithi070:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-31T02:52:53.241 DEBUG:teuthology.orchestra.run.smithi102:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-31T02:52:53.266 INFO:teuthology.orchestra.run.smithi070.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.0.log 2024-08-31T02:52:53.267 INFO:teuthology.orchestra.run.smithi070.stderr:: No such file or directory 2024-08-31T02:52:53.267 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/tcmu-runner.log 2024-08-31T02:52:53.267 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.1.log 2024-08-31T02:52:53.267 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.0.log: /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log 2024-08-31T02:52:53.267 INFO:teuthology.orchestra.run.smithi070.stderr: 86.3% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/tcmu-runner.log.gz 2024-08-31T02:52:53.271 INFO:teuthology.orchestra.run.smithi102.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.7.log 2024-08-31T02:52:53.271 INFO:teuthology.orchestra.run.smithi102.stderr:: No such file or directory 2024-08-31T02:52:53.271 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log 2024-08-31T02:52:53.271 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.4.log 2024-08-31T02:52:53.271 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log 2024-08-31T02:52:53.272 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log 2024-08-31T02:52:53.272 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.4.log: /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.6.log 2024-08-31T02:52:53.273 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log 2024-08-31T02:52:53.273 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.c.log 2024-08-31T02:52:53.273 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi070.usqphe.log/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log: 2024-08-31T02:52:53.273 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log 2024-08-31T02:52:53.274 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi070.usqphe.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.a.log 2024-08-31T02:52:53.274 INFO:teuthology.orchestra.run.smithi070.stderr: 62.8% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi070.usqphe.log.gz 2024-08-31T02:52:53.274 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.2.log 2024-08-31T02:52:53.274 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.y.log 2024-08-31T02:52:53.274 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.5.log 2024-08-31T02:52:53.276 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.6.log: 96.5% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log.gz 2024-08-31T02:52:53.279 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log 2024-08-31T02:52:53.279 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.5.log: 92.1% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log.gz 2024-08-31T02:52:53.281 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.2.log: 97.6% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.cephadm.log.gz 2024-08-31T02:52:53.281 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log 2024-08-31T02:52:53.286 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.y.log: 95.1% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.audit.log.gz 2024-08-31T02:52:53.286 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.3.log 2024-08-31T02:52:53.287 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi102.axbapt.log 2024-08-31T02:52:53.292 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log: 90.6% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log.gz 2024-08-31T02:52:53.295 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log: 94.7% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph.log.gz 2024-08-31T02:52:53.296 INFO:teuthology.orchestra.run.smithi070.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-31T02:52:53.301 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.x.log 2024-08-31T02:52:53.301 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi102.axbapt.log: 61.9% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-client.rgw.foo.smithi102.axbapt.log.gz 2024-08-31T02:52:53.301 INFO:teuthology.orchestra.run.smithi102.stderr:gzip -5 --verbose -- /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.b.log 2024-08-31T02:52:53.314 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-31T02:52:53.333 INFO:teuthology.orchestra.run.smithi070.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log.gz 2024-08-31T02:52:53.346 INFO:teuthology.orchestra.run.smithi102.stderr:/var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 92.4% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-volume.log.gz 2024-08-31T02:52:53.368 INFO:teuthology.orchestra.run.smithi070.stderr: 91.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-31T02:52:53.397 INFO:teuthology.orchestra.run.smithi102.stderr: 90.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-31T02:52:53.694 INFO:teuthology.orchestra.run.smithi102.stderr: 90.2% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.x.log.gz 2024-08-31T02:52:53.860 INFO:teuthology.orchestra.run.smithi070.stderr: 88.4% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mgr.y.log.gz 2024-08-31T02:52:54.054 INFO:teuthology.orchestra.run.smithi070.stderr: 92.5% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.c.log.gz 2024-08-31T02:52:54.180 INFO:teuthology.orchestra.run.smithi102.stderr: 92.6% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.b.log.gz 2024-08-31T02:52:54.591 INFO:teuthology.orchestra.run.smithi070.stderr: 93.2% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.2.log.gz 2024-08-31T02:52:54.655 INFO:teuthology.orchestra.run.smithi070.stderr: 93.3% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.0.log.gz 2024-08-31T02:52:54.679 INFO:teuthology.orchestra.run.smithi102.stderr: 92.9% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.6.log.gz 2024-08-31T02:52:54.834 INFO:teuthology.orchestra.run.smithi102.stderr: 93.7% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.7.log.gz 2024-08-31T02:52:54.846 INFO:teuthology.orchestra.run.smithi102.stderr: 93.2% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.5.log.gz 2024-08-31T02:52:54.946 INFO:teuthology.orchestra.run.smithi102.stderr: 93.4% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.4.log.gz 2024-08-31T02:52:54.948 INFO:teuthology.orchestra.run.smithi102.stderr: 2024-08-31T02:52:54.948 INFO:teuthology.orchestra.run.smithi102.stderr:real 0m1.692s 2024-08-31T02:52:54.948 INFO:teuthology.orchestra.run.smithi102.stderr:user 0m7.319s 2024-08-31T02:52:54.948 INFO:teuthology.orchestra.run.smithi102.stderr:sys 0m0.316s 2024-08-31T02:52:55.177 INFO:teuthology.orchestra.run.smithi070.stderr: 93.2% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.3.log.gz 2024-08-31T02:52:55.198 INFO:teuthology.orchestra.run.smithi070.stderr: 93.3% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-osd.1.log.gz 2024-08-31T02:52:55.267 INFO:teuthology.orchestra.run.smithi070.stderr: 91.6% -- replaced with /var/log/ceph/862b0c56-673e-11ef-bcd4-c7b262605968/ceph-mon.a.log.gz 2024-08-31T02:52:55.269 INFO:teuthology.orchestra.run.smithi070.stderr: 2024-08-31T02:52:55.269 INFO:teuthology.orchestra.run.smithi070.stderr:real 0m2.014s 2024-08-31T02:52:55.269 INFO:teuthology.orchestra.run.smithi070.stderr:user 0m9.661s 2024-08-31T02:52:55.269 INFO:teuthology.orchestra.run.smithi070.stderr:sys 0m0.359s 2024-08-31T02:52:55.269 INFO:tasks.cephadm:Archiving logs... 2024-08-31T02:52:55.270 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi070/log 2024-08-31T02:52:55.270 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-31T02:52:56.590 DEBUG:teuthology.misc:Transferring archived files from smithi102:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi102/log 2024-08-31T02:52:56.591 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-31T02:52:57.625 INFO:tasks.cephadm:Removing cluster... 2024-08-31T02:52:57.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 --force 2024-08-31T02:52:57.911 DEBUG:teuthology.orchestra.run.smithi102:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 --force 2024-08-31T02:52:58.194 INFO:tasks.cephadm:Removing cephadm ... 2024-08-31T02:52:58.194 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-31T02:52:58.210 DEBUG:teuthology.orchestra.run.smithi102:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-31T02:52:58.226 INFO:tasks.cephadm:Teardown complete 2024-08-31T02:52:58.226 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-31T02:52:58.234 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-31T02:52:58.235 DEBUG:teuthology.orchestra.run.smithi070:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T02:52:58.253 DEBUG:teuthology.orchestra.run.smithi102:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T02:52:58.265 INFO:teuthology.orchestra.run.smithi070.stderr:bash: line 1: ntpq: command not found 2024-08-31T02:52:58.269 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T02:52:58.269 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2024-08-31T02:52:58.270 INFO:teuthology.orchestra.run.smithi070.stdout:^* hv01.front.sepia.ceph.com 3 6 377 6 +335us[ +335us] +/- 47ms 2024-08-31T02:52:58.270 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 9 -1124us[-1123us] +/- 56ms 2024-08-31T02:52:58.270 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 11 +943us[ +944us] +/- 73ms 2024-08-31T02:52:58.270 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:52:58.280 INFO:teuthology.orchestra.run.smithi102.stderr:bash: line 1: ntpq: command not found 2024-08-31T02:52:58.284 INFO:teuthology.orchestra.run.smithi102.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T02:52:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:=============================================================================== 2024-08-31T02:52:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:^* hv01.front.sepia.ceph.com 3 6 377 11 +319us[ +327us] +/- 47ms 2024-08-31T02:52:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 11 -1132us[-1124us] +/- 56ms 2024-08-31T02:52:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 12 +950us[ +957us] +/- 73ms 2024-08-31T02:52:58.285 INFO:teuthology.orchestra.run.smithi102.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T02:52:58.286 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-31T02:52:58.294 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-31T02:52:58.295 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-31T02:52:58.303 DEBUG:teuthology.orchestra.run.smithi070:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-31T02:52:58.389 DEBUG:teuthology.orchestra.run.smithi102:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-31T02:52:58.458 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T02:52:58.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:52:58.487 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T02:52:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:52:58.515 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-31T02:52:58.524 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-31T02:52:58.531 INFO:teuthology.task.internal:Duration was 2513.780181 seconds 2024-08-31T02:52:58.531 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-31T02:52:58.543 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-31T02:52:58.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-31T02:52:58.545 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-31T02:52:58.582 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T02:52:58.594 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T02:52:58.958 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-31T02:52:58.959 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi070.front.sepia.ceph.com 2024-08-31T02:52:58.959 DEBUG:teuthology.orchestra.run.smithi070:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-31T02:52:58.981 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi102.front.sepia.ceph.com 2024-08-31T02:52:58.981 DEBUG:teuthology.orchestra.run.smithi102:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-31T02:52:59.026 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-31T02:52:59.026 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-31T02:52:59.029 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-31T02:52:59.093 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-31T02:52:59.093 DEBUG:teuthology.orchestra.run.smithi070:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-31T02:52:59.096 DEBUG:teuthology.orchestra.run.smithi102:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-31T02:52:59.944 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-31T02:52:59.955 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-31T02:52:59.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-31T02:52:59.980 DEBUG:teuthology.orchestra.run.smithi102:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-31T02:53:00.006 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-31T02:53:00.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-31T02:53:00.023 DEBUG:teuthology.orchestra.run.smithi102:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-31T02:53:00.044 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = core 2024-08-31T02:53:00.068 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = core 2024-08-31T02:53:00.104 DEBUG:teuthology.orchestra.run.smithi070:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-31T02:53:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:53:00.135 DEBUG:teuthology.orchestra.run.smithi102:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-31T02:53:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T02:53:00.159 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-31T02:53:00.168 INFO:teuthology.task.internal:Transferring archived files... 2024-08-31T02:53:00.169 DEBUG:teuthology.misc:Transferring archived files from smithi070:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi070 2024-08-31T02:53:00.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-31T02:53:00.231 DEBUG:teuthology.misc:Transferring archived files from smithi102:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-30_21:08:02-orch-squid-distro-default-smithi/7882562/remote/smithi102 2024-08-31T02:53:00.231 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-31T02:53:00.276 INFO:teuthology.task.internal:Removing archive directory... 2024-08-31T02:53:00.276 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-31T02:53:00.279 DEBUG:teuthology.orchestra.run.smithi102:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-31T02:53:00.324 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-31T02:53:00.337 INFO:teuthology.task.internal:Not uploading archives. 2024-08-31T02:53:00.337 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-31T02:53:00.347 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-31T02:53:00.347 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-31T02:53:00.350 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-31T02:53:00.362 INFO:teuthology.orchestra.run.smithi070.stdout: 262217 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 31 02:53 /home/ubuntu/cephtest 2024-08-31T02:53:00.379 INFO:teuthology.orchestra.run.smithi102.stdout: 262217 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 31 02:53 /home/ubuntu/cephtest 2024-08-31T02:53:00.381 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-31T02:53:00.390 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-31T02:53:00.412 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/connectivity} duration: 2513.7801806926727 failure_reason: 'Command failed on smithi070 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 862b0c56-673e-11ef-bcd4-c7b262605968 -e sha1=34b9d27bc925f169ace27b9f1f204536aa49fc0b -- bash -c \''ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e \''"\''"\''.up_to_date | length == 7\''"\''"\''\''''' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b20128395b0341bc9ef0ba1e668a21ad status: fail success: false 2024-08-31T02:53:00.412 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-31T02:53:00.500 INFO:teuthology.run:FAIL